Author SHA1 Message Date
Victor Shyba 71fb06fbb8 sendfile proof of concept 2021-08-02 03:23:54 -03:00
Niko e1b4f21e00 Merge pull request #52 from lbryio/ittt
Merge months of work including all sort of caches and O_DIRECT optimizations
2021-07-23 20:35:22 -04:00
Niko Storni b4913ecedf cleanup 2021-07-24 01:03:51 +02:00
Niko Storni 63a574ec2f unify caches
fix tests
2021-07-24 01:03:51 +02:00
Niko Storni b8af3408e0 move server packages 2021-07-24 01:03:51 +02:00
Niko Storni 847089d0d6 fix error propagation
update readme
2021-07-24 01:03:51 +02:00
Niko Storni 170dfef3a8 fix copy pasta mistake 2021-07-24 01:03:51 +02:00
Niko Storni 2b458a6bd0 fix params
more cleanups
2021-07-24 01:03:51 +02:00
Niko Storni febfc51cb0 refactor refactor refactor 2021-07-24 01:03:51 +02:00
Andrey Beletsky 72be487262 Fix broken import 2021-07-24 01:03:51 +02:00
Andrey Beletsky 94e7d81bd3 Fix OpenFile call flags for macos 2021-07-24 01:03:51 +02:00
Niko Storni c6c779da39 fix panic
fix counter leak
2021-07-24 01:03:51 +02:00
Niko Storni 2e101083e6 write blobs to tmp dir to avoid corruption 2021-07-24 01:03:51 +02:00
Niko Storni 63aacd8a69 use O_DIRECT to write to disk (fixes everything)
add queue back to serving blobs
improve a lot of things
upgrade modules
2021-07-24 01:03:51 +02:00
Niko Storni c03ae6487d fix unsafe dereference 2021-07-24 01:03:51 +02:00
Niko Storni 0c4f455f0c add metrics 2021-07-24 01:03:51 +02:00
Niko Storni af3e08c446 update lbry.go dep 2021-07-24 01:03:51 +02:00
Niko Storni 975bfe7fac upgrade singleflight
http store fix
2021-07-24 01:03:51 +02:00
Niko Storni b075d948bb remove locks causing deadlocks 2021-07-24 01:03:51 +02:00
Niko Storni 2651a64dbb add http server/client 2021-07-24 01:03:51 +02:00
Mark Beamer Jr fa7150cf2b Add queue to prevent writing too many files at once. 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 6c4db980c9 Add queue to prevent writing too many files at once. 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 7adaa510fd Add locks to disk store. 2021-07-24 01:03:51 +02:00
Niko Storni 64ed7304f6 add a lot of extra heavy debugging 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 5aefaf061e Add single flight for cache not just origin 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 724ee47c8b add metric calls for other packages 2021-07-24 01:03:51 +02:00
Mark Beamer Jr caaec6fcb1 add guage metrics for go routines in reflector package 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 15984b8fd9 add gops to reflector server 2021-07-24 01:03:51 +02:00
Niko Storni 2be913b077 request queue size param 2021-07-24 01:03:51 +02:00
Niko Storni 34c11b0a0e increase window size 2021-07-24 01:03:51 +02:00
Niko Storni 64acdc29c3 improve disk cleanup
add index to is_stored
fix test
replace LRU cache
2021-07-24 01:03:51 +02:00
Niko Storni 598773c90d fix mess with lbry.go 2021-07-24 01:03:47 +02:00
Niko Storni 766238fd7e add if this than that store
switch to wasabi for uploads
2021-07-24 01:03:47 +02:00
Niko Storni ac5242f173 add integrity check cmd
throttle live integrity checks
bug fixes
2021-07-24 01:03:47 +02:00
Mark Beamer Jr 215103cb33 use wait group not stopper 2021-07-24 01:03:47 +02:00
Mark Beamer Jr ed3622d0a6 Wait for request to be handled before returning 2021-07-24 01:03:47 +02:00
Mark Beamer Jr 848fce5afa Add request queue for blob cache 2021-07-24 01:03:47 +02:00
Niko Storni e37eeba0c9 check blobs when reading them 2021-07-24 01:03:47 +02:00
Niko Storni 7da49a4ccb upgrade quic-go
add cache for blobs not found
2021-07-24 01:03:47 +02:00
Niko Storni 7b02ace5e2 fix issues caused by beamer's renaming 2021-07-24 01:03:47 +02:00
Niko Storni 5fb67b32db run go mod tidy 2021-07-24 01:03:47 +02:00
Niko Storni a0c9ed2ace make it simpler 2021-07-24 01:03:47 +02:00
Niko Storni 998b082a06 remove panics 2021-07-24 01:03:47 +02:00
Niko Storni 36d4156e2a add tracing to blobs 2021-07-24 01:03:47 +02:00
Niko Storni 74925ebba2 optimize batch insertions
reduce touch time to every 6 hours
2021-07-24 01:03:47 +02:00
Alex Grintsvayg 6f95b3395f avoid heavy interpolateparams call 2021-07-24 01:03:47 +02:00
Alex Grintsvayg dff00e2317 fix long query 2021-07-24 01:03:47 +02:00
Alex Grintsvayg 9a5d9d7ff5 only touch blobs when you get them 2021-07-24 01:03:47 +02:00
Niko Storni 5794c57898 save uploaded blobs and work around the blocklist issue 2021-07-24 01:03:47 +02:00
Niko Storni 35c713a26e add cmd to populate db
fix store init
try fixing unreasonable db bottleneck
2021-07-24 01:03:47 +02:00
Alex Grintsvayg 6fb0620091 something like this 2021-07-24 01:03:38 +02:00
Niko Storni 03df751bc7 add PoC for litedb to avoid all the overhead 2021-07-24 01:03:16 +02:00
Niko Storni c902858958 address some review comments 2021-07-24 01:03:16 +02:00
Niko Storni 84fabdd5f4 add option to run with RO-CF only as upstream
increase idle timeout to avoid errors downstream
add option to delete blobs from DB if storage doesn't have it (for future local tracking)
2021-07-24 01:03:16 +02:00
Niko Storni f5cad15f84 upgrade quic 2021-07-24 01:03:10 +02:00
Niko Storni dd3d0ae42c update lfuda library 2021-07-24 01:03:10 +02:00
Niko Storni 0b565852b8 only store the blobs in the underlying storage if LFUDA accepted them 2021-07-24 01:03:10 +02:00
Niko Storni ff13d7b2f7 fix cache size mess 2021-07-24 01:03:10 +02:00
Niko Storni 7f5a89fa5a fix buffer cache running out of space 2021-07-24 01:03:10 +02:00
Niko Storni 704e15f8c1 use LFUDA store
swap size to bytes
2021-07-24 01:03:10 +02:00
Niko Storni 5eb1f13b54 add LFUDA store
update quic
fix tests
2021-07-24 01:03:10 +02:00
Alex Grintsvayg 176e05714e rename cahces 2021-07-24 01:03:10 +02:00
Niko Storni eefd84b02d add buffer cache for nvme drive 2021-07-24 01:03:10 +02:00
Niko Storni af2742c34f update quic
don't wait for a blob to be written to disk before sending it downstream
don't wait for the disk store to be walked before starting everything up
2021-07-24 01:03:10 +02:00
Alex Grintsvayg 2cf4acdb59 add 'send' command to efficiently send a file to reflector 2021-04-02 14:30:36 -04:00
Alex Grintsvayg 044e2fe5d7 Merge branch 'fix_mem_leak'
* fix_mem_leak:
  dont over-allocate ram when reading blobs via QUIC
  dont overallocate ram when reading blobs from disk
2020-11-27 16:18:44 -05:00
Alex Grintsvayg fb77bf621e dont over-allocate ram when reading blobs via QUIC 2020-11-27 16:18:26 -05:00
Alex Grintsvayg e70b9af3e4 dont overallocate ram when reading blobs from disk
ReadFile checks the file size and allocates a bit more space than we
expect we'll need. ReadAll uses Go's standard resizing algo, which
doubles the underlying array each time you hit the end. So ReadAll
ends up allocating 4MB for a full blob, while ReadFile allocates
slightly over 2MB.
2020-11-27 16:18:26 -05:00
Alex Grintsvayg 5df05cf46f ensure autodeploy is off 2020-11-23 12:41:09 -05:00
Alex Grintsvayg 7bddcf01b8 Merge branch 'smarter_caches'
* smarter_caches:
  use speedwalk for faster file listing
  remove afero fs abstraction in prep for using speedwalk
  add faster file.Walk fn. meant for DiskStore.list()
  clarify Get() error requirement
  separate singleflight cache wrapper, component names for cache metrics
  apparently the normal check doesn't work
  split cloudfront into RO and RW stores
  add noop store that does nothing
  add test for DiskStore.list()
  add lru cache eviction metric
  rename the stores, add caching to reflector cmd
  separate disk and lru behavior
  LRU cache for disk store, abstract fs in disk store for testing
2020-11-04 16:04:16 -05:00
Alex Grintsvayg 659a6e73cc use speedwalk for faster file listing 2020-11-04 16:03:47 -05:00
Alex Grintsvayg aaae3ffa5b remove afero fs abstraction in prep for using speedwalk 2020-11-04 16:03:47 -05:00
Alex Grintsvayg 131fed28d2 add faster file.Walk fn. meant for DiskStore.list() 2020-11-04 16:03:47 -05:00
Alex Grintsvayg 72571236ab clarify Get() error requirement 2020-11-04 16:03:46 -05:00
Alex Grintsvayg 560e180e36 separate singleflight cache wrapper, component names for cache metrics 2020-11-04 16:03:46 -05:00
Alex Grintsvayg 070c378dfd apparently the normal check doesn't work 2020-11-04 16:03:46 -05:00
Alex Grintsvayg 124d4065c2 split cloudfront into RO and RW stores 2020-11-04 16:03:45 -05:00
Alex Grintsvayg f131c1f35b add noop store that does nothing 2020-11-04 16:03:45 -05:00
Alex Grintsvayg 7a3225434e add test for DiskStore.list() 2020-11-04 16:03:45 -05:00
Alex Grintsvayg 3608971f0b add lru cache eviction metric 2020-11-04 16:03:44 -05:00
Alex Grintsvayg c9fa04043c rename the stores, add caching to reflector cmd 2020-11-04 16:03:44 -05:00
Alex Grintsvayg c6b53792c8 separate disk and lru behavior 2020-11-04 16:03:44 -05:00
Alex Grintsvayg 69fa06420b LRU cache for disk store, abstract fs in disk store for testing 2020-11-04 16:03:43 -05:00
Niko Storni 5cb1365903 increase idle timeout 2020-10-19 23:42:40 +02:00
Niko Storni e430c2fd40 bump go version 2020-10-15 03:43:03 +02:00
Niko 66024716ac Merge pull request #45 from lbryio/thundering_herd
ensure only single origin Get request is in flight per hash. protects against thundering herd.
2020-10-15 03:16:34 +02:00
Niko Storni f043516a14 revert upload changes 2020-10-15 03:08:27 +02:00
Niko Storni c3db95a6c1 add more metrics
increase handshake timeout by 1 second
2020-10-15 02:59:12 +02:00
Niko Storni 08c93d44fd update quic lib 2020-10-15 00:13:16 +02:00
Alex Grintsvayg b02e80d472 ensure only single origin Get request is in flight per hash. protects against thundering herd. 2020-10-15 00:12:31 +02:00
Niko Storni f1875454cc store uploaded blobs 2020-10-15 00:12:18 +02:00
Alex Grintsvayg a80e0f5b0d change error msg 2020-10-06 10:17:50 -04:00
Alex Grintsvayg 2b3581a692 silence all those quic errors 2020-10-06 10:11:36 -04:00
Alex Grintsvayg a0f78028cc handle the case where last_accessed_at is null 2020-10-06 09:39:43 -04:00
Alex Grintsvayg a084330055 drop mediainfo requirement since travis cant install it 2020-10-05 18:28:41 -04:00
Alex Grintsvayg ea80ed6506 install mediainfo during build 2020-10-05 18:22:33 -04:00
Alex Grintsvayg 4b335ed692 bumping to 1.15 for the quic-go dependency 2020-10-05 18:15:02 -04:00
Alex Grintsvayg 6118dde36c Merge branch 'track_access'
* track_access:
  small changes for easier testing
  track approximate access time for blobs
2020-10-05 18:10:49 -04:00
Alex Grintsvayg 8364d3fc54 small changes for easier testing 2020-10-05 18:08:54 -04:00
Alex Grintsvayg de0ccd4da7 track approximate access time for blobs 2020-10-05 18:08:53 -04:00
Alex Grintsvayg 31f9346027 Merge branch 'publish_from_go'
* publish_from_go:
  add publish command
2020-10-05 18:08:22 -04:00
Alex Grintsvayg 5658fe4607 add publish command 2020-10-05 16:30:57 -04:00
Niko Storni fc5f5ff7d3 add cloudfront support 2020-09-09 00:18:07 +02:00
Niko Storni e8b98bc862 update dependencies 2020-09-01 20:15:34 +02:00
Niko Storni f458529c74 fix memory leak? 2020-07-14 00:20:58 +02:00
Niko Storni 150b1f6f1f fix travis 2020-07-10 15:24:48 +02:00
Niko Storni 47f28002ff close unclosed handle
reduce idle timeout for http3
update QUIC library
2020-07-10 15:19:57 +02:00
Niko beff466d18 Merge pull request #43 from lbryio/http3
Add/replace peer protocol with HTTP3/QUIC protocol
see previous commits for more details
2020-07-09 15:17:35 +02:00
Niko Storni 694bda105c add metrics 2020-07-09 15:02:32 +02:00
Niko Storni 34ca7847d0 return a better descriptive error when the blob isn't found 2020-07-09 15:02:32 +02:00
Niko Storni 5c91051b78 disable disk cleanup routine 2020-07-09 15:02:32 +02:00
Alex Grintsvayg df4f42db82 successfully shut down wallet server 2020-07-09 15:02:32 +02:00
Alex Grintsvayg 41d758ef5c test wallet server connection 2020-07-09 15:02:32 +02:00
Niko Storni 264390a2b2 add debug code 2020-07-09 15:02:32 +02:00
Niko Storni fdcc41829a handshake changes 2020-07-09 15:02:32 +02:00
Niko Storni 09c7718f30 refactor code 2020-07-09 15:02:32 +02:00
Niko Storni 8a5f57b14f remove QUIC protocol
add HTTP3/QUIC protocol
2020-07-09 15:02:32 +02:00
Niko Storni 90997b9918 use new connections for each blob 2020-07-09 15:02:32 +02:00
Niko Storni e0da2674a1 use new connections for each action
reduce timeout
2020-07-09 15:02:32 +02:00
Niko Storni a80599413c improve params description 2020-07-09 15:02:32 +02:00
Niko Storni 3ffe7a10c7 add other reflector store
add flags
improve disk cleanup
2020-07-09 15:02:32 +02:00
Andrey Beletsky 1bf3cb81b3 Use ModTime on systems that don't provide Atim file stat field 2020-07-09 15:02:32 +02:00
Mark Beamer Jr 4a9f127ecc Add gops to reflector 2020-07-09 15:02:32 +02:00
Mark Beamer Jr de1fb63a1c fix potentially missing client or stream for a quic store 2020-07-09 15:02:32 +02:00
Mark Beamer Jr e98794e125 Add close function for quic store so routines do not hang waiting for timeout. 2020-07-09 15:02:32 +02:00
Niko Storni fb0004bac4 increase packet size
add retro-compatibility
remove unused SPV servers
remove travis test failure
2020-07-09 15:02:32 +02:00
Andrey Beletsky 1f12b1d5d8 Add dockerfile 2020-05-06 00:36:35 +07:00
Andrey Beletsky bc95ca61d5 Merge pull request #40 from lbryio/delete-after-upload
add flag to delete blobs after upload
2020-05-05 13:36:41 +07:00
Alex Grintsvayg be69c2f05c add flag to delete blobs after upload 2020-05-02 14:31:10 -04:00
Alex Grintsvayg 4a902597df thats confusing 2020-03-30 16:05:08 -04:00
Alex Grintsvayg dd528f03f6 expose raw wallet server request fn 2020-03-30 15:57:53 -04:00
Alex Grintsvayg e6ba61fce2 more info on json errors 2020-03-20 10:15:41 -04:00
Niko Storni e03abad012 Merge branch 'disk-cache' 2020-03-06 11:24:33 -05:00
Alex Grintsvayg dde93a1fe6 make cache dir a cli flag 2020-02-27 14:53:33 -05:00
Niko Storni 4a5a148843 implement disk cleanup 2020-02-27 14:53:33 -05:00
Niko Storni cb6443fa1e update spv servers 2020-02-27 14:53:27 -05:00
Alex Grintsvayg 1a1f991e48 more error traces 2020-02-26 10:02:53 -05:00
Alex Grintsvayg d1063bd54e log full traces more 2020-02-25 18:22:22 -05:00
Alex Grintsvayg d291c063ec add traces to unmarshall errors 2020-02-25 15:49:51 -05:00
Thomas Zarebczan a939529d19 Merge pull request #35 from ykris45/patch-1
Update LICENSE
2020-02-03 17:11:25 -05:00
YULIUS KURNIAWAN KRISTIANTO cf34547b8a Update LICENSE 2020-02-03 05:58:24 +07:00
Alex Grintsvayg 6631ad325f more errors 2020-01-13 16:16:46 -05:00
Alex Grintsvayg df266f6194 track hash mismatches 2020-01-07 08:38:40 -05:00
Alex Grintsvayg b3b581c00e more granular errors 2020-01-03 14:02:00 -05:00
Alex Grintsvayg 18b15674d9 warn on untracked error 2020-01-03 12:28:01 -05:00
Alex Grintsvayg 780e899e90 more specific errors 2020-01-03 10:27:29 -05:00
Alex Grintsvayg 5c5643749e i/o timeout is not the same as context.DeadlineExceeded 2020-01-02 13:58:11 -05:00
Alex Grintsvayg 11e50a6022 remove unused stats code 2020-01-02 13:27:34 -05:00
Alex Grintsvayg 1603b3bb22 fix circular import 2020-01-02 13:22:51 -05:00
Alex Grintsvayg 5465527faf Merge branch 'metrics'
* metrics:
  use labels for different error types
  switch to prometheus for metrics
2020-01-02 13:13:29 -05:00
Alex Grintsvayg 5d8a2d697c use labels for different error types 2020-01-02 13:12:33 -05:00
Alex Grintsvayg 50089481fb switch to prometheus for metrics 2020-01-02 13:12:33 -05:00
Alex Grintsvayg d6cf9e9e63 new uber-go import path 2019-12-28 19:24:26 -05:00
Alex Grintsvayg dc6dd8d12b fix blocklist, log future blocklist errors, add resolve wallet server method 2019-12-28 19:17:52 -05:00
Alex Grintsvayg ad8d623863 work around shitty wallet server error response 2019-12-28 19:14:24 -05:00
sayplastic d89b58c52a Merge pull request #31 from lbryio/lbry_tv
Put back printf statement into the stats module
2019-12-17 00:36:56 +07:00
Andrey Beletsky 2aa40850b0 Put back printf statement into the stats module 2019-12-17 00:33:34 +07:00
sayplastic ca24c2be28 Merge pull request #30 from lbryio/lbry_tv
upgrade to lbry.go v2.4.0
2019-12-16 22:53:00 +07:00
Andrey Beletsky 52127eee7c Remove setting default client timeout in peer.NewStore 2019-12-16 22:47:35 +07:00
Andrey Beletsky 95eb94f5a7 Remove excessive INFO-level logging 2019-12-16 21:52:51 +07:00
Andrey Beletsky 834733b675 Add options for peer.NewStore to allow for setting TCP timeout 2019-11-22 18:56:20 +07:00
Mark Beamer Jr 86a553b876 upgrade to lbry.go v2.4.0 2019-11-13 19:11:35 -05:00
Alex Grin e5438713ce Merge pull request #27 from StrikerRUS/patch-2
bump year in license
2019-10-09 10:37:39 -04:00
Nikita Titov f6c4b7d36b bump year in license 2019-10-08 23:32:22 +03:00
Alex Grintsvayg 661c20a21d make db-backed store more generic (not specific to s3) 2019-10-03 16:58:17 -04:00
Alex Grintsvayg 69f1e0f4ca make MemoryStore consistent with the New...() pattern 2019-10-03 16:49:28 -04:00
Alex Grintsvayg 2ca83139df use stream.Blob for BlobStore interface 2019-10-03 16:34:57 -04:00
Alex Grintsvayg 0af6d65d40 rename FileStore -> DiskStore 2019-10-03 16:24:59 -04:00
Alex Grintsvayg a8230db802 rewrite getstream command using caching store 2019-10-03 16:13:08 -04:00
Alex Grintsvayg 24f885e268 store blobs on disk in prefix-based subdirectories to avoid too many files in one dir 2019-10-03 16:12:49 -04:00
Alex Grintsvayg acb9840871 add peer blob store, which gets blobs from a peer 2019-10-03 16:10:29 -04:00
Alex Grintsvayg a745bafc58 go 1.13 2019-10-03 14:12:42 -04:00
Alex Grintsvayg bf9e90f14c comment out github releasing for clarity 2019-10-03 14:02:00 -04:00
Alex Grintsvayg a98d1e1217 tidy go.mod 2019-10-03 13:41:02 -04:00
Alex Grintsvayg 36ee7e8d1f add caching blob store 2019-10-03 13:36:35 -04:00
Alex Grintsvayg c1e8e7481f fix getstream command to write as it downloads (or else it runs out of memory) 2019-09-27 14:38:49 -04:00
Alex Grintsvayg cb669eb1a7 add claim decode command 2019-09-11 14:35:25 -04:00
Alex Grintsvayg 08df3b167c add getstream command to download a stream from a peer 2019-09-11 13:28:58 -04:00
Alex Grintsvayg 1a6b862c96 error on invalid blob hash length 2019-09-10 17:18:44 -04:00
Niko Storni ce3d4403db update lbry.go dependency 2019-08-28 15:16:02 +02:00
Niko Storni 949ea2f2d8 Merge branch 'public-lib' 2019-08-28 15:07:52 +02:00
Niko Storni 390d2b1eec export fields 2019-08-28 15:07:14 +02:00
Niko Storni 557c595e90 make summary available externally 2019-08-28 15:07:14 +02:00
Niko Storni 7d8772ed63 go mod tidy 2019-08-28 15:02:19 +02:00
Niko Storni 80f3b3437a re-enable reflector on new db 2019-08-20 01:26:23 +02:00
Niko Storni 68caed8a62 disable reflector for server replacement 2019-08-19 19:00:34 +02:00
Niko Storni 3823ee7a4e re-enable db 2019-08-08 19:49:32 +02:00
Niko Storni 5916ab4efd disable db temporarily 2019-08-08 18:42:46 +02:00
Niko Storni 227b362296 re-enable db 2019-08-07 21:32:00 +02:00
Niko Storni 8f280bf52f turn off db for instance resize 2019-08-07 20:39:27 +02:00
Alex Grintsvayg f30401d7a6 Revert "disable db for migration"
This reverts commit 83bb1791b4.
2019-07-30 18:25:08 -04:00
Alex Grintsvayg 83bb1791b4 disable db for migration 2019-07-30 18:10:01 -04:00
Alex Grintsvayg 3e006f1571 Revert "make new changes backwards-compatible while we migrate"
This reverts commit 594a45a271.
2019-07-30 18:03:56 -04:00
Alex Grintsvayg 2497f354bd update api domain 2019-07-30 18:02:32 -04:00
Alex Grintsvayg db83ebe434 update blocklist to new wallet servers 2019-07-30 17:07:01 -04:00
Niko Storni a2d51cee3d enable db 2019-07-10 20:59:52 +02:00
80 changed files with 6919 additions and 1057 deletions

No files matched your search

+10 -10
View File
@@ -6,7 +6,7 @@ env:
- GO111MODULE=on
go:
- 1.11.x
- 1.15.x
cache:
directories:
@@ -24,7 +24,7 @@ install: true
# set -e enabled in bash.
before_script:
# All the .go files, excluding vendor/ and model (auto generated)
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
# - go get github.com/golang/lint/golint # Linter
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
@@ -73,11 +73,11 @@ deploy:
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
bucket: "lbry-reflector-binary"
- provider: releases
file: ./bin/prism-bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
tags: true
api_key:
secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
# - provider: releases
# file: ./bin/prism-bin
# skip_cleanup: true
# on:
# repo: lbryio/reflector.go
# tags: true
# api_key:
# secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
+9
View File
@@ -0,0 +1,9 @@
FROM alpine
EXPOSE 8080
RUN mkdir /app
WORKDIR /app
COPY bin/prism-bin ./prism
RUN chmod +x prism
ENTRYPOINT [ "/app/prism" ]
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2016-2018 LBRY Inc
Copyright (c) 2016-2020 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+3 -3
View File
@@ -6,9 +6,9 @@ import (
"sort"
"time"
"github.com/lbryio/lbry.go/extras/crypto"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/crypto"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/hashicorp/serf/serf"
log "github.com/sirupsen/logrus"
+2 -1
View File
@@ -6,7 +6,8 @@ import (
"strconv"
"syscall"
"github.com/lbryio/lbry.go/extras/crypto"
"github.com/lbryio/lbry.go/v2/extras/crypto"
"github.com/lbryio/reflector.go/cluster"
log "github.com/sirupsen/logrus"
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"encoding/hex"
"fmt"
"github.com/lbryio/lbry.go/v2/schema/stake"
"github.com/davecgh/go-spew/spew"
"github.com/gogo/protobuf/jsonpb"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "decode VALUE",
Short: "Decode a claim value",
Args: cobra.ExactArgs(1),
Run: decodeCmd,
}
rootCmd.AddCommand(cmd)
}
func decodeCmd(cmd *cobra.Command, args []string) {
c, err := stake.DecodeClaimHex(args[0], "")
if err != nil {
log.Fatal(err)
}
m := jsonpb.Marshaler{Indent: " "}
if stream := c.Claim.GetStream(); stream != nil {
json, err := m.MarshalToString(stream)
if err != nil {
log.Fatal(err)
}
fmt.Println(json)
fmt.Printf("SD hash as hex: %s\n", hex.EncodeToString(stream.GetSource().GetSdHash()))
} else if channel := c.Claim.GetChannel(); channel != nil {
json, err := m.MarshalToString(channel)
if err != nil {
log.Fatal(err)
}
fmt.Println(json)
} else if repost := c.Claim.GetRepost(); repost != nil {
json, err := m.MarshalToString(repost)
if err != nil {
log.Fatal(err)
}
fmt.Println(json)
} else {
spew.Dump(c)
}
}
+2 -2
View File
@@ -8,8 +8,8 @@ import (
"syscall"
"time"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
+80
View File
@@ -0,0 +1,80 @@
package cmd
import (
"encoding/hex"
"os"
"time"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "getstream ADDRESS:PORT SDHASH",
Short: "Get a stream from a reflector server",
Args: cobra.ExactArgs(2),
Run: getStreamCmd,
}
rootCmd.AddCommand(cmd)
}
func getStreamCmd(cmd *cobra.Command, args []string) {
addr := args[0]
sdHash := args[1]
s := store.NewCachingStore(
"getstream",
peer.NewStore(peer.StoreOpts{Address: addr}),
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
)
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
var sd stream.SDBlob
sdb, _, err := s.Get(sdHash)
if err != nil {
log.Fatal(err)
}
err = sd.FromBlob(sdb)
if err != nil {
log.Fatal(err)
}
filename := sd.SuggestedFileName
if filename == "" {
filename = "stream_" + time.Now().Format("20060102_150405")
}
f, err := os.Create(wd + "/" + filename)
if err != nil {
log.Fatal(err)
}
for i := 0; i < len(sd.BlobInfos)-1; i++ {
b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
if err != nil {
log.Fatal(err)
}
data, err := b.Plaintext(sd.Key, sd.BlobInfos[i].IV)
if err != nil {
log.Fatal(err)
}
_, err = f.Write(data)
if err != nil {
log.Fatal(err)
}
}
}
+94
View File
@@ -0,0 +1,94 @@
package cmd
import (
"crypto/sha512"
"encoding/hex"
"io/ioutil"
"os"
"path"
"runtime"
"sync/atomic"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var threads int
func init() {
var cmd = &cobra.Command{
Use: "check-integrity",
Short: "check blobs integrity for a given path",
Run: integrityCheckCmd,
}
cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
rootCmd.AddCommand(cmd)
}
func integrityCheckCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
if diskStorePath == "" {
log.Fatal("store-path must be defined")
}
blobs, err := speedwalk.AllFiles(diskStorePath, true)
if err != nil {
log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
}
tasks := make(chan string, len(blobs))
done := make(chan bool)
processed := new(int32)
go produce(tasks, blobs)
cpus := runtime.NumCPU()
for i := 0; i < cpus-1; i++ {
go consume(i, tasks, done, len(blobs), processed)
}
<-done
}
func produce(tasks chan<- string, blobs []string) {
for _, b := range blobs {
tasks <- b
}
close(tasks)
}
func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
start := time.Now()
for b := range tasks {
processedSoFar := atomic.AddInt32(processed, 1)
if worker == 0 {
remaining := int32(totalTasks) - processedSoFar
timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
}
blobPath := path.Join(diskStorePath, b[:2], b)
blob, err := ioutil.ReadFile(blobPath)
if err != nil {
if os.IsNotExist(err) {
continue
}
log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
continue
}
hashBytes := sha512.Sum384(blob)
readHash := hex.EncodeToString(hashBytes[:])
if readHash != b {
log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
err := os.Remove(blobPath)
if err != nil {
log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
}
}
}
done <- true
}
+18 -7
View File
@@ -7,30 +7,41 @@ import (
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var peerNoDB bool
func init() {
var cmd = &cobra.Command{
Use: "peer",
Short: "Run peer server",
Run: peerCmd,
}
cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
rootCmd.AddCommand(cmd)
}
func peerCmd(cmd *cobra.Command, args []string) {
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
checkErr(err)
var err error
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
combo := store.NewDBBackedS3Store(s3, db)
peerServer := peer.NewServer(combo)
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
peerServer := peer.NewServer(s3)
if !peerNoDB {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = db.Connect(globalConfig.DBConn)
checkErr(err)
combo := store.NewDBBackedStore(s3, db, false)
peerServer = peer.NewServer(combo)
}
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
if err != nil {
+51
View File
@@ -0,0 +1,51 @@
package cmd
import (
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
diskStorePath string
)
func init() {
var cmd = &cobra.Command{
Use: "populate-db",
Short: "populate local database with blobs from a disk storage",
Run: populateDbCmd,
}
cmd.Flags().StringVar(&diskStorePath, "store-path", "",
"path of the store where all blobs are cached")
rootCmd.AddCommand(cmd)
}
func populateDbCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
if diskStorePath == "" {
log.Fatal("store-path must be defined")
}
localDb := &db.SQL{
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
blobs, err := speedwalk.AllFiles(diskStorePath, true)
if err != nil {
log.Fatal(err)
}
err = localDb.AddBlobs(blobs)
if err != nil {
log.Errorf("error while storing to db: %s", errors.FullTrace(err))
}
}
+65
View File
@@ -0,0 +1,65 @@
package cmd
import (
"fmt"
"github.com/lbryio/reflector.go/publish"
"github.com/lbryio/lbry.go/v2/lbrycrd"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "publish FILE",
Short: "Publish a file",
Args: cobra.ExactArgs(1),
Run: publishCmd,
}
cmd.Flags().String("name", "", "Claim name")
cmd.Flags().String("title", "", "Title of the content")
cmd.Flags().String("description", "", "Description of the content")
cmd.Flags().String("author", "", "Content author")
cmd.Flags().String("tags", "", "Comma-separated list of tags")
cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
rootCmd.AddCommand(cmd)
}
func publishCmd(cmd *cobra.Command, args []string) {
var err error
claimName := mustGetFlagString(cmd, "name")
if claimName == "" {
log.Errorln("--name required")
return
}
path := args[0]
client, err := lbrycrd.NewWithDefaultURL(nil)
checkErr(err)
tx, txid, err := publish.Publish(
client,
path,
claimName,
"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
publish.Details{
Title: mustGetFlagString(cmd, "title"),
Description: mustGetFlagString(cmd, "description"),
Author: mustGetFlagString(cmd, "author"),
Tags: nil,
ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
},
"reflector.lbry.com:5566",
)
checkErr(err)
decoded, err := publish.Decode(client, tx)
checkErr(err)
fmt.Printf("TX: %s\n\n", decoded)
fmt.Printf("TXID: %s\n", txid.String())
}
+334 -39
View File
@@ -4,81 +4,376 @@ import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/http"
http32 "github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
//port configuration
tcpPeerPort int
http3PeerPort int
httpPeerPort int
receiverPort int
metricsPort int
//flags configuration
disableUploads bool
disableBlocklist bool
useDB bool
//upstream configuration
upstreamReflector string
upstreamProtocol string
//downstream configuration
requestQueueSize int
//upstream edge configuration (to "cold" storage)
originEndpoint string
originEndpointFallback string
//cache configuration
diskCache string
secondaryDiskCache string
memCache int
)
var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
var cacheMangerToGcache = map[string]store.EvictionStrategy{
"lfu": store.LFU,
"arc": store.ARC,
"lru": store.LRU,
"simple": store.SIMPLE,
}
func init() {
var cmd = &cobra.Command{
Use: "reflector",
Short: "Run reflector server",
Run: reflectorCmd,
}
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
rootCmd.AddCommand(cmd)
}
func reflectorCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339))
log.Printf("reflector %s", meta.VersionString())
// flip this flag to false when doing db maintenance. uploads will not work (as reflector server wont be running)
// but downloads will still work straight from s3
useDB := false
// the blocklist logic requires the db backed store to be the outer-most store
underlyingStore := initStores()
underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
var err error
var blobStore store.BlobStore = s3
var reflectorServer *reflector.Server
if useDB {
db := new(db.SQL)
err = db.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
blobStore = store.NewDBBackedS3Store(s3, db)
reflectorServer = reflector.NewServer(blobStore)
if !disableUploads {
reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
reflectorServer.Timeout = 3 * time.Minute
if globalConfig.SlackHookURL != "" {
reflectorServer.StatLogger = log.StandardLogger()
reflectorServer.StatReportFrequency = 1 * time.Hour
}
reflectorServer.EnableBlocklist = true
reflectorServer.EnableBlocklist = !disableBlocklist
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
if err != nil {
log.Fatal(err)
}
defer reflectorServer.Shutdown()
}
peerServer := peer.NewServer(blobStore)
if globalConfig.SlackHookURL != "" {
peerServer.StatLogger = log.StandardLogger()
peerServer.StatReportFrequency = 1 * time.Hour
}
err = peerServer.Start(":5567")
peerServer := peer.NewServer(underlyingStoreWithCaches)
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
if err != nil {
log.Fatal(err)
}
defer peerServer.Shutdown()
http3PeerServer := http32.NewServer(underlyingStoreWithCaches, requestQueueSize)
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
if err != nil {
log.Fatal(err)
}
defer http3PeerServer.Shutdown()
httpServer := http.NewServer(underlyingStoreWithCaches, requestQueueSize)
err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
if err != nil {
log.Fatal(err)
}
defer httpServer.Shutdown()
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
defer underlyingStoreWithCaches.Shutdown()
defer underlyingStore.Shutdown() //do we actually need this? Oo
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
if reflectorServer != nil {
reflectorServer.Shutdown()
// deferred shutdowns happen now
cleanerStopper.StopAndWait()
}
func initUpstreamStore() store.BlobStore {
var s store.BlobStore
if upstreamReflector == "" {
return nil
}
switch upstreamProtocol {
case "tcp":
s = peer.NewStore(peer.StoreOpts{
Address: upstreamReflector,
Timeout: 30 * time.Second,
})
case "http3":
s = http32.NewStore(http32.StoreOpts{
Address: upstreamReflector,
Timeout: 30 * time.Second,
})
case "http":
s = store.NewHttpStore(upstreamReflector)
default:
log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
}
return s
}
func initEdgeStore() store.BlobStore {
var s3Store *store.S3Store
var s store.BlobStore
if conf != "none" {
s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
}
if originEndpointFallback != "" && originEndpoint != "" {
ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
if s3Store != nil {
s = store.NewCloudFrontRWStore(ittt, s3Store)
} else {
s = ittt
}
} else if s3Store != nil {
s = s3Store
} else {
log.Fatalf("this configuration does not include a valid upstream source")
}
return s
}
func initDBStore(s store.BlobStore) store.BlobStore {
if useDB {
dbInst := &db.SQL{
TrackAccess: db.TrackAccessStreams,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := dbInst.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
s = store.NewDBBackedStore(s, dbInst, false)
}
return s
}
func initStores() store.BlobStore {
s := initUpstreamStore()
if s == nil {
s = initEdgeStore()
}
s = initDBStore(s)
return s
}
// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
stopper := stop.New()
diskStore := initDiskStore(s, diskCache, stopper)
finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
stop.New()
if memCache > 0 {
finalStore = store.NewCachingStore(
"reflector",
finalStore,
store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
)
}
return finalStore, stopper
}
func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
if diskCacheMaxSize == 0 {
return upstreamStore
}
err := os.MkdirAll(diskCachePath, os.ModePerm)
if err != nil {
log.Fatal(err)
}
diskStore := store.NewDiskStore(diskCachePath, 2)
var unwrappedStore store.BlobStore
cleanerStopper := stop.New(stopper)
if cacheManager == "localdb" {
localDb := &db.SQL{
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
} else {
unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
}
wrapped := store.NewCachingStore(
"reflector",
upstreamStore,
unwrappedStore,
)
return wrapped
}
func diskCacheParams(diskParams string) (int, string, string) {
if diskParams == "" {
return 0, "", ""
}
parts := strings.Split(diskParams, ":")
if len(parts) != 3 {
log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
}
diskCacheSize := parts[0]
path := parts[1]
cacheManager := parts[2]
if len(path) == 0 || path[0] != '/' {
log.Fatalf("disk cache paths must start with '/'")
}
if !util.InSlice(cacheManager, cacheManagers) {
log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
}
var maxSize datasize.ByteSize
err := maxSize.UnmarshalText([]byte(diskCacheSize))
if err != nil {
log.Fatal(err)
}
if maxSize <= 0 {
log.Fatal("disk cache size must be more than 0")
}
return int(maxSize), path, cacheManager
}
func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
// this is so that it runs on startup without having to wait for 10 minutes
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-stopper.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
blobsCount, err := db.Count()
if err != nil {
return err
}
if blobsCount >= maxItems {
itemsToDelete := blobsCount / 10
blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := &stop.Group{}
go func() {
for _, hash := range blobs {
select {
case <-stopper.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-stopper.Ch():
return
default:
}
err = store.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
+50
View File
@@ -0,0 +1,50 @@
package cmd
import (
"encoding/hex"
"encoding/json"
"fmt"
"github.com/lbryio/reflector.go/wallet"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "resolve ADDRESS:PORT URL",
Short: "Resolve a URL",
Args: cobra.ExactArgs(2),
Run: resolveCmd,
}
rootCmd.AddCommand(cmd)
}
func resolveCmd(cmd *cobra.Command, args []string) {
addr := args[0]
url := args[1]
node := wallet.NewNode()
defer node.Shutdown()
err := node.Connect([]string{addr}, nil)
checkErr(err)
output, err := node.Resolve(url)
checkErr(err)
claim, err := node.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
checkErr(err)
jsonClaim, err := json.MarshalIndent(claim, "", " ")
checkErr(err)
fmt.Println(string(jsonClaim))
}
func rev(b []byte) []byte {
r := make([]byte, len(b))
for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
r[left], r[right] = b[right], b[left]
}
return r
}
+29 -6
View File
@@ -4,12 +4,14 @@ import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/reflector.go/updater"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/johntdyer/slackrus"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -49,7 +51,7 @@ var rootCmd = &cobra.Command{
func init() {
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config")
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
}
// Execute adds all child commands to the root command and sets flags appropriately.
@@ -68,8 +70,11 @@ func preRun(cmd *cobra.Command, args []string) {
debugLogger.SetOutput(os.Stderr)
if util.InSlice(verboseAll, verbose) {
logrus.Info("global verbose logging enabled")
logrus.SetLevel(logrus.DebugLevel)
verbose = []string{verboseDHT, verboseNodeFinder}
} else if len(verbose) > 0 {
logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
}
for _, debugType := range verbose {
@@ -141,9 +146,27 @@ func loadConfig(path string) (Config, error) {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, err
return c, errors.Err(err)
}
err = json.Unmarshal(raw, &c)
return c, err
return c, errors.Err(err)
}
func mustGetFlagString(cmd *cobra.Command, name string) string {
v, err := cmd.Flags().GetString(name)
checkErr(err)
return v
}
func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
v, err := cmd.Flags().GetInt64(name)
checkErr(err)
return v
}
//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
// v, err := cmd.Flags().GetBool(name)
// checkErr(err)
// return v
//}
+160
View File
@@ -0,0 +1,160 @@
package cmd
import (
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/signal"
"path"
"syscall"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "send ADDRESS:PORT PATH",
Short: "Send a file to a reflector",
Args: cobra.ExactArgs(2),
Run: sendCmd,
}
cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
rootCmd.AddCommand(cmd)
}
// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
// re-uploading blobs that already exist
var hackyReflector reflector.Client
func sendCmd(cmd *cobra.Command, args []string) {
reflectorAddress := args[0]
err := hackyReflector.Connect(reflectorAddress)
checkErr(err)
defer hackyReflector.Close()
filePath := args[1]
file, err := os.Open(filePath)
checkErr(err)
defer file.Close()
sdCachePath := ""
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
if sdCacheDir != "" {
if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
err = os.MkdirAll(sdCacheDir, 0777)
checkErr(err)
}
sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
}
var enc *stream.Encoder
if sdCachePath != "" {
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
sdBlob, err := ioutil.ReadFile(sdCachePath)
checkErr(err)
cachedSDBlob := &stream.SDBlob{}
err = cachedSDBlob.FromBlob(sdBlob)
checkErr(err)
enc = stream.NewEncoderFromSD(file, cachedSDBlob)
}
}
if enc == nil {
enc = stream.NewEncoder(file)
}
exitCode := 0
var killed bool
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
go func() {
sig := <-interruptChan
fmt.Printf("caught %s, exiting...\n", sig.String())
killed = true
exitCode = 1
}()
for {
if killed {
break
}
b, err := enc.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
fmt.Printf("error reading next blob: %v\n", err)
exitCode = 1
break
}
err = hackyReflect(b, false)
if err != nil {
fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
exitCode = 1
break
}
}
sd := enc.SDBlob()
//sd.StreamName = filepath.Base(filePath)
//sd.SuggestedFileName = filepath.Base(filePath)
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
if err != nil {
fmt.Printf("error saving sd blob: %v\n", err)
fmt.Println(sd.ToJson())
exitCode = 1
}
if killed {
os.Exit(exitCode)
}
if reflectorAddress != "" {
err = hackyReflect(sd.ToBlob(), true)
if err != nil {
fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
exitCode = 1
}
}
ret := struct {
SDHash string `json:"sd_hash"`
SourceHash string `json:"source_hash"`
}{
SDHash: sd.HashHex(),
SourceHash: hex.EncodeToString(enc.SourceHash()),
}
j, err := json.MarshalIndent(ret, "", " ")
checkErr(err)
fmt.Println(string(j))
os.Exit(exitCode)
}
func hackyReflect(b stream.Blob, sd bool) error {
var err error
if sd {
err = hackyReflector.SendSDBlob(b)
} else {
err = hackyReflector.SendBlob(b)
}
if errors.Is(err, reflector.ErrBlobExists) {
//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
return nil
}
return err
}
+37 -10
View File
@@ -2,10 +2,11 @@ package cmd
import (
"crypto/rand"
"os"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/stream"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -13,9 +14,9 @@ import (
func init() {
var cmd = &cobra.Command{
Use: "sendblob ADDRESS:PORT",
Use: "sendblob ADDRESS:PORT [PATH]",
Short: "Send a random blob to a reflector server",
Args: cobra.ExactArgs(1),
Args: cobra.RangeArgs(1, 2),
Run: sendBlobCmd,
}
rootCmd.AddCommand(cmd)
@@ -23,6 +24,10 @@ func init() {
func sendBlobCmd(cmd *cobra.Command, args []string) {
addr := args[0]
var path string
if len(args) >= 2 {
path = args[1]
}
c := reflector.Client{}
err := c.Connect(addr)
@@ -30,14 +35,36 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
log.Fatal("error connecting client to server: ", err)
}
blob := make(stream.Blob, 1024)
_, err = rand.Read(blob)
if err != nil {
log.Fatal("failed to make random blob: ", err)
if path == "" {
blob := make(stream.Blob, 1024)
_, err = rand.Read(blob)
if err != nil {
log.Fatal("failed to make random blob: ", err)
}
err = c.SendBlob(blob)
if err != nil {
log.Error(err)
}
return
}
err = c.SendBlob(blob)
if err != nil {
log.Error(err)
file, err := os.Open(path)
checkErr(err)
defer file.Close()
s, err := stream.New(file)
checkErr(err)
sdBlob := &stream.SDBlob{}
err = sdBlob.FromBlob(s[0])
checkErr(err)
for i, b := range s {
if i == 0 {
err = c.SendSDBlob(b)
} else {
err = c.SendBlob(b)
}
checkErr(err)
}
}
+9 -6
View File
@@ -7,15 +7,16 @@ import (
"strings"
"syscall"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/prism"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@@ -52,11 +53,13 @@ func init() {
}
func startCmd(cmd *cobra.Command, args []string) {
db := new(db.SQL)
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
comboStore := store.NewDBBackedS3Store(s3, db)
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
comboStore := store.NewDBBackedStore(s3, db, false)
conf := prism.DefaultConf()
+4 -4
View File
@@ -9,8 +9,8 @@ import (
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
@@ -27,11 +27,11 @@ func init() {
}
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s", meta.Version)
log.Printf("reflector %s", meta.VersionString())
memStore := &store.MemoryBlobStore{}
memStore := store.NewMemStore()
reflectorServer := reflector.NewServer(memStore)
reflectorServer := reflector.NewServer(memStore, memStore)
reflectorServer.Timeout = 3 * time.Minute
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
+10 -5
View File
@@ -9,11 +9,13 @@ import (
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var uploadWorkers int
var uploadSkipExistsCheck bool
var uploadDeleteBlobsAfterUpload bool
func init() {
var cmd = &cobra.Command{
@@ -24,19 +26,22 @@ func init() {
}
cmd.PersistentFlags().IntVar(&uploadWorkers, "workers", 1, "How many worker threads to run at once")
cmd.PersistentFlags().BoolVar(&uploadSkipExistsCheck, "skipExistsCheck", false, "Dont check if blobs exist before uploading")
cmd.PersistentFlags().BoolVar(&uploadDeleteBlobsAfterUpload, "deleteBlobsAfterUpload", false, "Delete blobs after uploading them")
rootCmd.AddCommand(cmd)
}
func uploadCmd(cmd *cobra.Command, args []string) {
db := new(db.SQL)
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := db.Connect(globalConfig.DBConn)
checkErr(err)
st := store.NewDBBackedS3Store(
store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db)
st := store.NewDBBackedStore(
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db, false)
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck)
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+2 -2
View File
@@ -2,9 +2,9 @@ package cmd
import (
"fmt"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/spf13/cobra"
)
@@ -18,5 +18,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Printf("version %s (built %s)\n", meta.Version, meta.BuildTime.Format(time.RFC3339))
fmt.Println(meta.VersionString())
}
+318 -131
View File
@@ -3,15 +3,22 @@ package db
import (
"context"
"database/sql"
"fmt"
"runtime"
"strings"
"time"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/lbry.go/extras/errors"
qt "github.com/lbryio/lbry.go/extras/query"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/lbry.go/v2/extras/errors"
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null"
"go.uber.org/atomic"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
@@ -29,17 +36,38 @@ type SdBlob struct {
StreamHash string `json:"stream_hash"`
}
type trackAccess int
const (
TrackAccessNone trackAccess = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
)
// SQL implements the DB interface
type SQL struct {
conn *sql.DB
// Track the approx last time a blob or stream was accessed
TrackAccess trackAccess
// Instead of deleting a blob, marked it as not stored in the db
SoftDelete bool
// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
LogQueries bool
}
func logQuery(query string, args ...interface{}) {
s, err := qt.InterpolateParams(query, args...)
func (s SQL) logQuery(query string, args ...interface{}) {
if !s.LogQueries {
return
}
qStr, err := qt.InterpolateParams(query, args...)
if err != nil {
log.Errorln(err)
} else {
log.Debugln(s)
log.Debugln(qStr)
}
}
@@ -70,15 +98,97 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
return err
}
//AddBlobs adds blobs to the database.
func (s *SQL) AddBlobs(hash []string) error {
if s.conn == nil {
return errors.Err("not connected")
}
batch := 10000
totalBlobs := int64(len(hash))
work := make(chan []string, 1000)
stopper := stop.New()
var totalInserted atomic.Int64
start := time.Now()
go func() {
for i := 0; i < len(hash); i += batch {
j := i + batch
if j > len(hash) {
j = len(hash)
}
work <- hash[i:j]
}
log.Infof("done loading %d hashes in the work queue", len(hash))
close(work)
}()
for i := 0; i < runtime.NumCPU(); i++ {
stopper.Add(1)
go func(worker int) {
log.Infof("starting worker %d", worker)
defer stopper.Done()
for hashes := range work {
inserted := totalInserted.Load()
remaining := totalBlobs - inserted
if inserted > 0 {
timePerBlob := time.Since(start).Microseconds() / inserted
remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
}
err := s.insertBlobs(hashes) // Process the batch.
if err != nil {
log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
}
totalInserted.Add(int64(len(hashes)))
}
}(i)
}
stopper.Wait()
return nil
}
func (s *SQL) insertBlobs(hashes []string) error {
var (
q string
//args []interface{}
)
dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
for _, hash := range hashes {
// prepared statements slow everything down by a lot due to reflection
// for this specific instance we'll go ahead and hardcode the query to make it go faster
q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
}
q = strings.TrimSuffix(q, ",")
_, err := s.exec(q)
if err != nil {
return err
}
return nil
}
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
}
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
hash, isStored, length,
var (
q string
args []interface{}
)
if s.TrackAccess == TrackAccessBlobs {
args = []interface{}{hash, isStored, length, time.Now()}
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
} else {
args = []interface{}{hash, isStored, length}
q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
}
blobID, err := s.exec(q, args...)
if err != nil {
return 0, err
}
@@ -91,16 +201,33 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
if blobID == 0 {
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
if s.TrackAccess == TrackAccessBlobs {
err := s.touchBlobs([]uint64{uint64(blobID)})
if err != nil {
return 0, errors.Err(err)
}
}
}
return blobID, nil
}
func (s *SQL) insertStream(hash, sdHash string, sdBlobID int64) (int64, error) {
streamID, err := s.exec(
"INSERT IGNORE INTO stream (hash, sd_hash, sd_blob_id) VALUES (?,?, ?)",
hash, sdHash, sdBlobID,
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
var (
q string
args []interface{}
)
if s.TrackAccess == TrackAccessStreams {
args = []interface{}{hash, sdBlobID, time.Now()}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
} else {
args = []interface{}{hash, sdBlobID}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
}
streamID, err := s.exec(q, args...)
if err != nil {
return 0, errors.Err(err)
}
@@ -113,13 +240,20 @@ func (s *SQL) insertStream(hash, sdHash string, sdBlobID int64) (int64, error) {
if streamID == 0 {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
if s.TrackAccess == TrackAccessStreams {
err := s.touchStreams([]uint64{uint64(streamID)})
if err != nil {
return 0, errors.Err(err)
}
}
}
return streamID, nil
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
exists, err := s.HasBlobs([]string{hash})
func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
exists, err := s.HasBlobs([]string{hash}, touch)
if err != nil {
return false, err
}
@@ -127,13 +261,72 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
if touch {
if s.TrackAccess == TrackAccessBlobs {
_ = s.touchBlobs(idsNeedingTouch)
} else if s.TrackAccess == TrackAccessStreams {
_ = s.touchStreams(idsNeedingTouch)
}
}
var hash string
return exists, err
}
func (s *SQL) touchBlobs(blobIDs []uint64) error {
if len(blobIDs) == 0 {
return nil
}
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
args := make([]interface{}, len(blobIDs)+1)
args[0] = time.Now()
for i := range blobIDs {
args[i+1] = blobIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) touchStreams(streamIDs []uint64) error {
if len(streamIDs) == 0 {
return nil
}
query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
args := make([]interface{}, len(streamIDs)+1)
args[0] = time.Now()
for i := range streamIDs {
args[i+1] = streamIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
if s.conn == nil {
return nil, nil, errors.Err("not connected")
}
var (
hash string
blobID uint64
streamID null.Uint64
lastAccessedAt null.Time
)
var needsTouch []uint64
exists := make(map[string]bool)
touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
maxBatchSize := 10000
doneIndex := 0
@@ -145,14 +338,29 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + qt.Qs(len(batch)) + ")"
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
args[i+1] = batch[i]
var query string
if s.TrackAccess == TrackAccessBlobs {
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else if s.TrackAccess == TrackAccessStreams {
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
FROM blob_ b
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else {
query = `SELECT b.hash, b.id, NULL, NULL
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
}
logQuery(query, args...)
args := make([]interface{}, len(batch))
for i := range batch {
args[i] = batch[i]
}
s.logQuery(query, args...)
err := func() error {
startTime := time.Now()
@@ -164,11 +372,18 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
defer closeRows(rows)
for rows.Next() {
err := rows.Scan(&hash)
err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
if s.TrackAccess == TrackAccessBlobs {
needsTouch = append(needsTouch, blobID)
} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
needsTouch = append(needsTouch, streamID.Uint64)
}
}
}
err = rows.Err()
@@ -180,16 +395,22 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
return nil
}()
if err != nil {
return nil, err
return nil, nil, err
}
}
return exists, nil
return exists, needsTouch, nil
}
// Delete will remove the blob from the db
// Delete will remove (or soft-delete) the blob from the db
// NOTE: If SoftDelete is enabled, streams will never be deleted
func (s *SQL) Delete(hash string) error {
_, err := s.exec("DELETE FROM stream WHERE sd_hash = ?", hash)
if s.SoftDelete {
_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
return errors.Err(err)
}
_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
if err != nil {
return errors.Err(err)
}
@@ -198,11 +419,59 @@ func (s *SQL) Delete(hash string) error {
return errors.Err(err)
}
//LeastRecentlyAccessedHashes gets the least recently accessed blobs
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
if s.TrackAccess != TrackAccessBlobs {
return nil, errors.Err("blob access tracking is disabled")
}
query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
s.logQuery(query, maxBlobs)
rows, err := s.conn.Query(query, maxBlobs)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
blobs := make([]string, 0, maxBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) Count() (int, error) {
if s.conn == nil {
return 0, errors.Err("not connected")
}
query := "SELECT count(id) from blob_"
if s.SoftDelete {
query += " where is_stored = 1"
}
s.logQuery(query)
var count int
err := s.conn.QueryRow(query).Scan(&count)
return count, errors.Err(err)
}
// Block will mark a blob as blocked
func (s *SQL) Block(hash string) error {
query := "INSERT IGNORE INTO blocked SET hash = ?"
args := []interface{}{hash}
logQuery(query, args...)
s.logQuery(query, args...)
_, err := s.conn.Exec(query, args...)
return errors.Err(err)
}
@@ -210,7 +479,7 @@ func (s *SQL) Block(hash string) error {
// GetBlocked will return a list of blocked hashes
func (s *SQL) GetBlocked() (map[string]bool, error) {
query := "SELECT hash FROM blocked"
logQuery(query)
s.logQuery(query)
rows, err := s.conn.Query(query)
if err != nil {
return nil, errors.Err(err)
@@ -246,13 +515,14 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
query := `
SELECT b.hash FROM blob_ b
INNER JOIN stream_blob sb ON b.hash = sb.blob_hash
INNER JOIN stream s ON s.hash = sb.stream_hash AND s.sd_hash = ?
INNER JOIN stream_blob sb ON b.id = sb.blob_id
INNER JOIN stream s ON s.id = sb.stream_id
INNER JOIN blob_ sdb ON sdb.id = s.sd_blob_id AND sdb.hash = ?
WHERE b.is_stored = 0
`
args := []interface{}{sdHash}
logQuery(query, args...)
s.logQuery(query, args...)
rows, err := s.conn.Query(query, args...)
if err != nil {
@@ -291,7 +561,7 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
return err
}
streamID, err := s.insertStream(sdBlob.StreamHash, sdHash, sdBlobID)
streamID, err := s.insertStream(sdBlob.StreamHash, sdBlobID)
if err != nil {
return err
}
@@ -308,9 +578,10 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
return err
}
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
_, err = s.exec(
"INSERT IGNORE INTO stream_blob (stream_id, stream_hash, blob_id, blob_hash, num) VALUES (?,?,?,?,?)",
streamID, sdBlob.StreamHash, blobID, contentBlob.BlobHash, contentBlob.BlobNum,
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
args...,
)
if err != nil {
return errors.Err(err)
@@ -330,7 +601,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
query := "SELECT MIN(hash), MAX(hash) from blob_"
logQuery(query)
s.logQuery(query)
err := s.conn.QueryRow(query).Scan(&min, &max)
return min, max, err
@@ -354,7 +625,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
args := []interface{}{start.Hex(), end.Hex()}
logQuery(query, args...)
s.logQuery(query, args...)
rows, err := s.conn.Query(query, args...)
defer closeRows(rows)
@@ -435,7 +706,7 @@ func closeRows(rows *sql.Rows) {
}
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
logQuery(query, args...)
s.logQuery(query, args...)
attempt, maxAttempts := 0, 3
Retry:
attempt++
@@ -473,17 +744,22 @@ CREATE TABLE blob_ (
hash char(96) NOT NULL,
is_stored TINYINT(1) NOT NULL DEFAULT 0,
length bigint(20) unsigned DEFAULT NULL,
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY blob_hash_idx (hash)
UNIQUE KEY blob_hash_idx (hash),
KEY `blob_last_accessed_idx` (`last_accessed_at`),
KEY `is_stored_idx` (`is_stored`)
);
CREATE TABLE stream (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
hash char(96) NOT NULL,
sd_blob_id BIGINT UNSIGNED NOT NULL,
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY stream_hash_idx (hash),
KEY stream_sd_blob_id_idx (sd_blob_id),
KEY last_accessed_at_idx (last_accessed_at),
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
);
@@ -502,93 +778,4 @@ CREATE TABLE blocked (
PRIMARY KEY (hash)
);
FOR THE MIGRATION, USE THESE TRIGGERS
CREATE TRIGGER tgr_stream_insert AFTER INSERT ON stream FOR EACH ROW INSERT INTO stream_new SET id = NEW.id, hash = NEW.hash, sd_blob_id = (SELECT id from blob_ where hash = NEW.sd_hash);
CREATE TRIGGER tgr_stream_delete AFTER DELETE ON stream FOR EACH ROW DELETE FROM stream_new WHERE id = OLD.id;
CREATE TRIGGER tgr_blob_insert AFTER INSERT ON blob_ FOR EACH ROW INSERT INTO blob_new SET id = NEW.id, hash = NEW.hash, is_stored = NEW.is_stored, length = NEW.length;
CREATE TRIGGER tgr_blob_update AFTER UPDATE ON blob_ FOR EACH ROW UPDATE blob_new SET hash = NEW.hash, is_stored = NEW.is_stored, length = NEW.length WHERE id = NEW.id;
CREATE TRIGGER tgr_blob_delete AFTER DELETE ON blob_ FOR EACH ROW DELETE FROM blob_new WHERE id = OLD.id;
CREATE TRIGGER tgr_stream_blob_insert AFTER INSERT ON stream_blob FOR EACH ROW INSERT INTO stream_blob_new SET stream_id = NEW.stream_id, blob_id = NEW.blob_id, num = NEW.num;
CREATE TRIGGER tgr_stream_blob_delete AFTER DELETE ON stream_blob FOR EACH ROW DELETE FROM stream_blob_new WHERE stream_id = OLD.stream_id AND blob_id = OLD.blob_id;
DROP PROCEDURE IF EXISTS copyblobs;
DELIMITER $$
CREATE PROCEDURE copyblobs()
BEGIN
DECLARE first_trigger_blob_id INT DEFAULT 124802284; # ID of first blob that was copied using the triggers. dont copy anything after that.
DECLARE i INT DEFAULT 0;
DECLARE minid BIGINT UNSIGNED DEFAULT 0;
SELECT min(id) INTO minid FROM blob_ WHERE id < first_trigger_blob_id AND id > (SELECT coalesce(max(id),0) from blob_new where id < first_trigger_blob_id);
wloop: WHILE minid is not null DO
#IF (i >= 100) THEN
# LEAVE wloop;
#END IF;
SET i = i + 1;
IF (i % 5000 = 0) THEN
SELECT concat('loop ', i, ', id ', minid) as progress;
END IF;
IF (i % 10 = 1) THEN # we start our loops on 1, like normal people
START TRANSACTION;
END IF;
INSERT INTO blob_new (id, hash, is_stored, length) SELECT id, hash, is_stored, length from blob_ where id = minid;
IF (i % 10 = 0) THEN
COMMIT;
END IF;
SELECT min(id) INTO minid FROM blob_ WHERE id < first_trigger_blob_id AND id > minid;
END WHILE wloop;
COMMIT;
END$$
DELIMITER ;
DROP PROCEDURE IF EXISTS copystreams;
DELIMITER $$
CREATE PROCEDURE copystreams()
BEGIN
DECLARE first_trigger_stream_id INT DEFAULT 1465749; # ID of first stream that was copied using the triggers. dont copy anything after that.
DECLARE i INT DEFAULT 0;
DECLARE minid BIGINT UNSIGNED DEFAULT 0;
DECLARE streamhash char(96);
SELECT min(id) INTO minid FROM stream WHERE id < first_trigger_stream_id AND id > (SELECT coalesce(max(id),0) from stream_new where id < first_trigger_stream_id);
wloop: WHILE minid is not null DO
#IF (i >= 10) THEN
# LEAVE wloop;
#END IF;
SET i = i + 1;
IF (i % 5000 = 0) THEN
SELECT concat('loop ', i, ', id ', minid) as progress;
END IF;
IF (i % 10 = 1) THEN # we start our loops on 1, like normal people
START TRANSACTION;
END IF;
SELECT hash INTO streamhash FROM stream WHERE id = minid;
INSERT INTO stream_new (id, hash, sd_blob_id) SELECT s.id, s.hash, b.id FROM stream s INNER JOIN blob_ b ON s.sd_hash = b.hash WHERE s.id = minid;
INSERT INTO stream_blob_new SELECT minid, b.id, sb.num FROM stream_blob sb INNER JOIN blob_ b ON sb.blob_hash = b.hash WHERE sb.stream_hash = streamhash;
IF (i % 10 = 0) THEN
COMMIT;
END IF;
SELECT min(id) INTO minid FROM stream WHERE id < first_trigger_stream_id AND id > minid;
END WHILE wloop;
COMMIT;
END$$
DELIMITER ;
*/
+43 -59
View File
@@ -1,67 +1,51 @@
module github.com/lbryio/reflector.go
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
cloud.google.com/go v0.34.0 // indirect
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78
github.com/aws/aws-sdk-go v1.16.11
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95 // indirect
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a
github.com/btcsuite/goleveldb v1.0.0 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.27.0
github.com/bluele/gcache v0.0.2
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/davecgh/go-spew v1.1.1
github.com/go-errors/errors v1.0.1
github.com/go-ini/ini v1.41.0
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706
github.com/golang/mock v1.2.0 // indirect
github.com/golang/protobuf v1.3.1
github.com/gorilla/context v1.1.1
github.com/gorilla/mux v1.6.2
github.com/gorilla/rpc v1.1.0
github.com/gorilla/websocket v1.4.0
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47
github.com/hashicorp/memberlist v0.1.0
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
github.com/gin-gonic/gin v1.7.1
github.com/go-sql-driver/mysql v1.6.0
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gops v0.3.18
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.9.5
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/inconshreveable/mousetrap v1.0.0
github.com/jessevdk/go-flags v1.4.0 // indirect
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5
github.com/miekg/dns v1.0.8
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/nlopes/slack v0.4.0
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c
github.com/karrick/godirwalk v1.16.1
github.com/lbryio/chainquery v1.9.0
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
github.com/lucas-clemente/quic-go v0.20.1
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/pkg/errors v0.8.1 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561
github.com/sergi/go-diff v1.0.0
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
github.com/sirupsen/logrus v1.3.0
github.com/prometheus/client_golang v1.10.0
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb
github.com/spf13/pflag v1.0.1
github.com/uber-go/atomic v1.3.2
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 // indirect
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034 // indirect
google.golang.org/appengine v1.4.0
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79
gopkg.in/yaml.v2 v2.2.2 // indirect
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea // indirect
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1 // indirect
github.com/stretchr/testify v1.7.0
github.com/volatiletech/null v8.0.0+incompatible
go.uber.org/atomic v1.7.0
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
)
go 1.16
+815 -105
View File
File diff suppressed because it is too large. Load diff
+324
View File
@@ -0,0 +1,324 @@
package metrics
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"syscall"
"time"
ee "github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
)
type Server struct {
srv *http.Server
stop *stop.Stopper
}
func NewServer(address string, path string) *Server {
h := http.NewServeMux()
h.Handle(path, promhttp.Handler())
return &Server{
srv: &http.Server{
Addr: address,
Handler: h,
//https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
//https://blog.cloudflare.com/exposing-go-on-the-internet/
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
},
stop: stop.New(),
}
}
func (s *Server) Start() {
s.stop.Add(1)
go func() {
defer s.stop.Done()
err := s.srv.ListenAndServe()
if err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Error(err)
}
}()
}
func (s *Server) Shutdown() {
_ = s.srv.Shutdown(context.Background())
s.stop.StopAndWait()
}
const (
ns = "reflector"
subsystemCache = "cache"
subsystemITTT = "ittt"
labelDirection = "direction"
labelErrorType = "error_type"
DirectionUpload = "upload" // to reflector
DirectionDownload = "download" // from reflector
LabelCacheType = "cache_type"
LabelComponent = "component"
LabelSource = "source"
errConnReset = "conn_reset"
errReadConnReset = "read_conn_reset"
errWriteConnReset = "write_conn_reset"
errReadConnTimedOut = "read_conn_timed_out"
errNoNetworkActivity = "no_network_activity"
errWriteConnTimedOut = "write_conn_timed_out"
errWriteBrokenPipe = "write_broken_pipe"
errEPipe = "e_pipe"
errETimedout = "e_timedout"
errIOTimeout = "io_timeout"
errUnexpectedEOF = "unexpected_eof"
errUnexpectedEOFStr = "unexpected_eof_str"
errJSONSyntax = "json_syntax"
errBlobTooBig = "blob_too_big"
errDeadlineExceeded = "deadline_exceeded"
errHashMismatch = "hash_mismatch"
errZeroByteBlob = "zero_byte_blob"
errInvalidCharacter = "invalid_character"
errBlobNotFound = "blob_not_found"
errNoErr = "no_error"
errQuicProto = "quic_protocol_violation"
errOther = "other"
)
var (
ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Name: "error_total",
Help: "Total number of errors",
}, []string{labelDirection, labelErrorType})
BlobDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "blob_download_total",
Help: "Total number of blobs downloaded from reflector",
})
PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "peer_download_total",
Help: "Total number of blobs downloaded from reflector through tcp protocol",
})
Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http3_blob_download_total",
Help: "Total number of blobs downloaded from reflector through QUIC protocol",
})
HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http_blob_download_total",
Help: "Total number of blobs downloaded from reflector through HTTP protocol",
})
CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "hit_total",
Help: "Total number of blobs retrieved from the cache storage",
}, []string{LabelCacheType, LabelComponent})
ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "this_hit_total",
Help: "Total number of blobs retrieved from the this storage",
})
ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "that_hit_total",
Help: "Total number of blobs retrieved from the that storage",
})
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "miss_total",
Help: "Total number of blobs retrieved from origin rather than cache storage",
}, []string{LabelCacheType, LabelComponent})
CacheOriginRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "origin_requests_total",
Help: "How many Get requests are in flight from the cache to the origin",
}, []string{LabelCacheType, LabelComponent})
//during thundering-herd situations, the metric below should be a lot smaller than the metric above
CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "waiting_requests_total",
Help: "How many cache requests are waiting for an in-flight origin request",
}, []string{LabelCacheType, LabelComponent})
CacheLRUEvictCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "evict_total",
Help: "Count of blobs evicted from cache",
}, []string{LabelCacheType, LabelComponent})
CacheRetrievalSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Name: "speed_mbps",
Help: "Speed of blob retrieval from cache or from origin",
}, []string{LabelCacheType, LabelComponent, LabelSource})
BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "blob_upload_total",
Help: "Total number of blobs uploaded to reflector",
})
SDBlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "sdblob_upload_total",
Help: "Total number of SD blobs (and therefore streams) uploaded to reflector",
})
MtrInBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "tcp_in_bytes",
Help: "Total number of bytes downloaded through TCP",
})
MtrOutBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "tcp_out_bytes",
Help: "Total number of bytes streamed out through TCP",
})
MtrInBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "udp_in_bytes",
Help: "Total number of bytes downloaded through UDP",
})
MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http_in_bytes",
Help: "Total number of bytes downloaded through HTTP",
})
MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "udp_out_bytes",
Help: "Total number of bytes streamed out through UDP",
})
MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http_out_bytes",
Help: "Total number of bytes streamed out through UDP",
})
MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "reflector_in_bytes",
Help: "Total number of incoming bytes (from users)",
})
MtrOutBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_out_bytes",
Help: "Total number of outgoing bytes (to S3)",
})
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_in_bytes",
Help: "Total number of incoming bytes (from S3-CF)",
})
Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: ns,
Name: "http3_blob_request_queue_size",
Help: "Blob requests of https queue size",
})
HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: ns,
Name: "http_blob_request_queue_size",
Help: "Blob requests queue size of the HTTP protocol",
})
RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Name: "routines",
Help: "routines running by type",
}, []string{"package", "kind"})
)
func CacheLabels(name, component string) prometheus.Labels {
return prometheus.Labels{
LabelCacheType: name,
LabelComponent: component,
}
}
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
return
}
err := ee.Wrap(e, 0)
errType := errOther
if strings.Contains(err.Error(), "i/o timeout") {
errType = errIOTimeout
} else if errors.Is(e, syscall.ECONNRESET) {
// Looks like we're getting this when direction == "download", but read_conn_reset and
// write_conn_reset when its "upload"
errType = errConnReset
} else if errors.Is(e, context.DeadlineExceeded) {
errType = errDeadlineExceeded
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
errType = errReadConnReset
} else if strings.Contains(err.Error(), "write: connection reset by peer") { // the other side closed the connection using TCP reset
errType = errWriteConnReset
} else if errors.Is(e, syscall.ETIMEDOUT) {
errType = errETimedout
} else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
errType = errReadConnTimedOut
} else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
errType = errNoNetworkActivity
} else if strings.Contains(err.Error(), "write: connection timed out") {
errType = errWriteConnTimedOut
} else if errors.Is(e, io.ErrUnexpectedEOF) {
errType = errUnexpectedEOF
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
errType = errUnexpectedEOFStr
} else if errors.Is(e, syscall.EPIPE) {
errType = errEPipe
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
// I believe this is the same as EPipe when direction == "download", but not for upload
errType = errWriteBrokenPipe
//} else if errors.Is(e, reflector.ErrBlobTooBig) { # this creates a circular import
// errType = errBlobTooBig
} else if strings.Contains(err.Error(), "blob must be at most") {
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
errType = errBlobTooBig
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
errType = errHashMismatch
} else if strings.Contains(err.Error(), "blob not found") {
errType = errBlobNotFound
} else if strings.Contains(err.Error(), "0-byte blob received") {
errType = errZeroByteBlob
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
errType = errQuicProto
} else if strings.Contains(err.Error(), "invalid character") {
errType = errInvalidCharacter
} else if _, ok := e.(*json.SyntaxError); ok {
errType = errJSONSyntax
} else if strings.Contains(err.Error(), "NO_ERROR") {
errType = errNoErr
} else {
log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction)
shouldLog = true
}
ErrorCount.With(map[string]string{
labelDirection: direction,
labelErrorType: errType,
}).Inc()
return
}
+348
View File
@@ -0,0 +1,348 @@
package lite_db
import (
"database/sql"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
}
// SQL implements the DB interface
type SQL struct {
conn *sql.DB
TrackAccessTime bool
}
func logQuery(query string, args ...interface{}) {
s, err := qt.InterpolateParams(query, args...)
if err != nil {
log.Errorln(err)
} else {
log.Debugln(s)
}
}
// Connect will create a connection to the database
func (s *SQL) Connect(dsn string) error {
var err error
// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
// opened inside a transaction. closing them manually doesn't seem to help
dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
s.conn, err = sql.Open("mysql", dsn)
if err != nil {
return errors.Err(err)
}
s.conn.SetMaxIdleConns(12)
return errors.Err(s.conn.Ping())
}
// AddBlob adds a blob to the database.
func (s *SQL) AddBlob(hash string, length int) error {
if s.conn == nil {
return errors.Err("not connected")
}
_, err := s.insertBlob(hash, length)
return err
}
func (s *SQL) insertBlob(hash string, length int) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
}
const isStored = true
now := time.Now()
args := []interface{}{hash, isStored, length, now}
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
args...,
)
if err != nil {
return 0, err
}
if blobID == 0 {
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
if err != nil {
return 0, errors.Err(err)
}
if blobID == 0 {
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
}
return blobID, nil
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
exists, err := s.HasBlobs([]string{hash})
if err != nil {
return false, err
}
return exists[hash], nil
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
s.touch(streamsNeedingTouch)
return exists, err
}
func (s *SQL) touch(blobIDs []uint64) error {
if len(blobIDs) == 0 {
return nil
}
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
args := make([]interface{}, len(blobIDs)+1)
args[0] = time.Now()
for i := range blobIDs {
args[i+1] = blobIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
if s.conn == nil {
return nil, nil, errors.Err("not connected")
}
var (
hash string
blobID uint64
lastAccessedAt null.Time
)
var needsTouch []uint64
exists := make(map[string]bool)
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
maxBatchSize := 10000
doneIndex := 0
for len(hashes) > doneIndex {
sliceEnd := doneIndex + maxBatchSize
if sliceEnd > len(hashes) {
sliceEnd = len(hashes)
}
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
query := `SELECT hash, id, last_accessed_at
FROM blob_
WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
args[i+1] = batch[i]
}
logQuery(query, args...)
err := func() error {
startTime := time.Now()
rows, err := s.conn.Query(query, args...)
log.Debugf("hashes query took %s", time.Since(startTime))
if err != nil {
return errors.Err(err)
}
defer closeRows(rows)
for rows.Next() {
err := rows.Scan(&hash, &blobID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
needsTouch = append(needsTouch, blobID)
}
}
err = rows.Err()
if err != nil {
return errors.Err(err)
}
doneIndex += len(batch)
return nil
}()
if err != nil {
return nil, nil, err
}
}
return exists, needsTouch, nil
}
// Delete will remove the blob from the db
func (s *SQL) Delete(hash string) error {
_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
return errors.Err(err)
}
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
// but they are tracked so reflector knows what it is missing.
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
if s.conn == nil {
return errors.Err("not connected")
}
_, err := s.insertBlob(sdHash, sdBlobLength)
return err
}
// GetHashRange gets the smallest and biggest hashes in the db
func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
const isStored = true
logQuery(query, isStored, maxBlobs)
rows, err := s.conn.Query(query, isStored, maxBlobs)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
blobs := make([]string, 0, maxBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) AllBlobs() ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
const isStored = true
logQuery(query, isStored)
rows, err := s.conn.Query(query, isStored)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
totalBlobs, err := s.BlobsCount()
if err != nil {
return nil, err
}
blobs := make([]string, 0, totalBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) BlobsCount() (int, error) {
if s.conn == nil {
return 0, errors.Err("not connected")
}
query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
const isStored = true
logQuery(query, isStored)
var count int
err := s.conn.QueryRow(query, isStored).Scan(&count)
return count, errors.Err(err)
}
func closeRows(rows *sql.Rows) {
if rows != nil {
err := rows.Close()
if err != nil {
log.Error("error closing rows: ", err)
}
}
}
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
logQuery(query, args...)
attempt, maxAttempts := 0, 3
Retry:
attempt++
result, err := s.conn.Exec(query, args...)
if isLockTimeoutError(err) {
if attempt <= maxAttempts {
//Error 1205: Lock wait timeout exceeded; try restarting transaction
goto Retry
}
err = errors.Prefix("Lock timeout for query "+query, err)
}
if err != nil {
return 0, errors.Err(err)
}
lastID, err := result.LastInsertId()
return lastID, errors.Err(err)
}
func isLockTimeoutError(err error) bool {
e, ok := err.(*mysql.MySQLError)
return ok && e != nil && e.Number == 1205
}
/* SQL schema
in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
CREATE TABLE `blob_` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`hash` char(96) NOT NULL,
`is_stored` tinyint(1) NOT NULL DEFAULT '0',
`length` bigint unsigned DEFAULT NULL,
`last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`),
UNIQUE KEY `blob_hash_idx` (`hash`),
KEY `blob_last_accessed_idx` (`last_accessed_at`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
*/
+6
View File
@@ -5,9 +5,15 @@ import (
"time"
"github.com/lbryio/reflector.go/cmd"
"github.com/google/gops/agent"
log "github.com/sirupsen/logrus"
)
func main() {
if err := agent.Listen(agent.Options{}); err != nil {
log.Fatal(err)
}
rand.Seed(time.Now().UnixNano())
cmd.Execute()
}
+19 -3
View File
@@ -1,6 +1,7 @@
package meta
import (
"fmt"
"strconv"
"time"
)
@@ -12,9 +13,24 @@ var BuildTime time.Time
func init() {
if Time != "" {
t, err := strconv.Atoi(Time)
if err != nil {
return
if err == nil {
BuildTime = time.Unix(int64(t), 0).UTC()
}
BuildTime = time.Unix(int64(t), 0).UTC()
}
}
func VersionString() string {
version := Version
if version == "" {
version = "<unset>"
}
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"
} else {
buildTime = BuildTime.Format(time.RFC3339)
}
return fmt.Sprintf("version %s, built %s", version, buildTime)
}
+6 -6
View File
@@ -5,16 +5,16 @@ import (
"strconv"
"sync"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
)
@@ -79,7 +79,7 @@ func New(conf *Config) *Prism {
dht: d,
cluster: c,
peer: peer.NewServer(conf.Blobs),
reflector: reflector.NewServer(conf.Blobs),
reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
grp: stop.New(),
}
+2 -1
View File
@@ -4,8 +4,9 @@ import (
"math/big"
"testing"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/lbry.go/dht/bits"
)
func TestAnnounceRange(t *testing.T) {
+174
View File
@@ -0,0 +1,174 @@
package publish
import "strings"
func guessMimeType(ext string) (string, string) {
if ext == "" {
return "application/octet-stream", "binary"
}
ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
types := map[string]struct{ mime, t string }{
"a": {"application/octet-stream", "binary"},
"ai": {"application/postscript", "image"},
"aif": {"audio/x-aiff", "audio"},
"aifc": {"audio/x-aiff", "audio"},
"aiff": {"audio/x-aiff", "audio"},
"au": {"audio/basic", "audio"},
"avi": {"video/x-msvideo", "video"},
"bat": {"text/plain", "document"},
"bcpio": {"application/x-bcpio", "binary"},
"bin": {"application/octet-stream", "binary"},
"bmp": {"image/bmp", "image"},
"c": {"text/plain", "document"},
"cdf": {"application/x-netcdf", "binary"},
"cpio": {"application/x-cpio", "binary"},
"csh": {"application/x-csh", "binary"},
"css": {"text/css", "document"},
"csv": {"text/csv", "document"},
"dll": {"application/octet-stream", "binary"},
"doc": {"application/msword", "document"},
"dot": {"application/msword", "document"},
"dvi": {"application/x-dvi", "binary"},
"eml": {"message/rfc822", "document"},
"eps": {"application/postscript", "document"},
"epub": {"application/epub+zip", "document"},
"etx": {"text/x-setext", "document"},
"exe": {"application/octet-stream", "binary"},
"gif": {"image/gif", "image"},
"gtar": {"application/x-gtar", "binary"},
"h": {"text/plain", "document"},
"hdf": {"application/x-hdf", "binary"},
"htm": {"text/html", "document"},
"html": {"text/html", "document"},
"ico": {"image/vnd.microsoft.icon", "image"},
"ief": {"image/ief", "image"},
"iges": {"model/iges", "model"},
"jpe": {"image/jpeg", "image"},
"jpeg": {"image/jpeg", "image"},
"jpg": {"image/jpeg", "image"},
"js": {"application/javascript", "document"},
"json": {"application/json", "document"},
"ksh": {"text/plain", "document"},
"latex": {"application/x-latex", "binary"},
"m1v": {"video/mpeg", "video"},
"m3u": {"application/vnd.apple.mpegurl", "audio"},
"m3u8": {"application/vnd.apple.mpegurl", "audio"},
"man": {"application/x-troff-man", "document"},
"markdown": {"text/markdown", "document"},
"md": {"text/markdown", "document"},
"me": {"application/x-troff-me", "binary"},
"mht": {"message/rfc822", "document"},
"mhtml": {"message/rfc822", "document"},
"mif": {"application/x-mif", "binary"},
"mov": {"video/quicktime", "video"},
"movie": {"video/x-sgi-movie", "video"},
"mp2": {"audio/mpeg", "audio"},
"mp3": {"audio/mpeg", "audio"},
"mp4": {"video/mp4", "video"},
"mpa": {"video/mpeg", "video"},
"mpe": {"video/mpeg", "video"},
"mpeg": {"video/mpeg", "video"},
"mpg": {"video/mpeg", "video"},
"ms": {"application/x-troff-ms", "binary"},
"nc": {"application/x-netcdf", "binary"},
"nws": {"message/rfc822", "document"},
"o": {"application/octet-stream", "binary"},
"obj": {"application/octet-stream", "model"},
"oda": {"application/oda", "binary"},
"p12": {"application/x-pkcs12", "binary"},
"p7c": {"application/pkcs7-mime", "binary"},
"pbm": {"image/x-portable-bitmap", "image"},
"pdf": {"application/pdf", "document"},
"pfx": {"application/x-pkcs12", "binary"},
"pgm": {"image/x-portable-graymap", "image"},
"pl": {"text/plain", "document"},
"png": {"image/png", "image"},
"pnm": {"image/x-portable-anymap", "image"},
"pot": {"application/vnd.ms-powerpoint", "document"},
"ppa": {"application/vnd.ms-powerpoint", "document"},
"ppm": {"image/x-portable-pixmap", "image"},
"pps": {"application/vnd.ms-powerpoint", "document"},
"ppt": {"application/vnd.ms-powerpoint", "document"},
"ps": {"application/postscript", "document"},
"pwz": {"application/vnd.ms-powerpoint", "document"},
"py": {"text/x-python", "document"},
"pyc": {"application/x-python-code", "binary"},
"pyo": {"application/x-python-code", "binary"},
"qt": {"video/quicktime", "video"},
"ra": {"audio/x-pn-realaudio", "audio"},
"ram": {"application/x-pn-realaudio", "audio"},
"ras": {"image/x-cmu-raster", "image"},
"rdf": {"application/xml", "binary"},
"rgb": {"image/x-rgb", "image"},
"roff": {"application/x-troff", "binary"},
"rtx": {"text/richtext", "document"},
"sgm": {"text/x-sgml", "document"},
"sgml": {"text/x-sgml", "document"},
"sh": {"application/x-sh", "document"},
"shar": {"application/x-shar", "binary"},
"snd": {"audio/basic", "audio"},
"so": {"application/octet-stream", "binary"},
"src": {"application/x-wais-source", "binary"},
"stl": {"model/stl", "model"},
"sv4cpio": {"application/x-sv4cpio", "binary"},
"sv4crc": {"application/x-sv4crc", "binary"},
"svg": {"image/svg+xml", "image"},
"swf": {"application/x-shockwave-flash", "binary"},
"t": {"application/x-troff", "binary"},
"tar": {"application/x-tar", "binary"},
"tcl": {"application/x-tcl", "binary"},
"tex": {"application/x-tex", "binary"},
"texi": {"application/x-texinfo", "binary"},
"texinfo": {"application/x-texinfo", "binary"},
"tif": {"image/tiff", "image"},
"tiff": {"image/tiff", "image"},
"tr": {"application/x-troff", "binary"},
"tsv": {"text/tab-separated-values", "document"},
"txt": {"text/plain", "document"},
"ustar": {"application/x-ustar", "binary"},
"vcf": {"text/x-vcard", "document"},
"wav": {"audio/x-wav", "audio"},
"webm": {"video/webm", "video"},
"wiz": {"application/msword", "document"},
"wsdl": {"application/xml", "document"},
"xbm": {"image/x-xbitmap", "image"},
"xlb": {"application/vnd.ms-excel", "document"},
"xls": {"application/vnd.ms-excel", "document"},
"xml": {"text/xml", "document"},
"xpdl": {"application/xml", "document"},
"xpm": {"image/x-xpixmap", "image"},
"xsl": {"application/xml", "document"},
"xwd": {"image/x-xwindowdump", "image"},
"zip": {"application/zip", "binary"},
// These are non-standard types, commonly found in the wild.
"cbr": {"application/vnd.comicbook-rar", "document"},
"cbz": {"application/vnd.comicbook+zip", "document"},
"flac": {"audio/flac", "audio"},
"lbry": {"application/x-ext-lbry", "document"},
"m4v": {"video/m4v", "video"},
"mid": {"audio/midi", "audio"},
"midi": {"audio/midi", "audio"},
"mkv": {"video/x-matroska", "video"},
"mobi": {"application/x-mobipocket-ebook", "document"},
"oga": {"audio/ogg", "audio"},
"ogv": {"video/ogg", "video"},
"pct": {"image/pict", "image"},
"pic": {"image/pict", "image"},
"pict": {"image/pict", "image"},
"prc": {"application/x-mobipocket-ebook", "document"},
"rtf": {"application/rtf", "document"},
"xul": {"text/xul", "document"},
// microsoft is special and has its own "standard"
// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
"wmv": {"video/x-ms-wmv", "video"},
}
if data, ok := types[ext]; ok {
return data.mime, data.t
}
return "application/x-ext-" + ext, "binary"
}
+292
View File
@@ -0,0 +1,292 @@
package publish
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"sort"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/lbry.go/v2/stream"
pb "github.com/lbryio/types/v2/go"
"github.com/btcsuite/btcd/btcjson"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
"github.com/golang/protobuf/proto"
)
/* TODO:
import cert from wallet
get all utxos from chainquery
create transaction
sign it with the channel
track state of utxos across publishes from this channel so that we can just do one query to get utxos
prioritize only confirmed utxos
Handling all the issues we handle currently with lbrynet:
"Couldn't find private key for id",
"You already have a stream claim published under the name",
"Cannot publish using channel",
"txn-mempool-conflict",
"too-long-mempool-chain",
"Missing inputs",
"Not enough funds to cover this transaction",
*/
type Details struct {
Title string
Description string
Author string
Tags []string
ReleaseTime int64
}
func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
if name == "" {
return nil, nil, errors.Err("name required")
}
//TODO: sign claim if publishing into channel
addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
if errors.Is(err, btcutil.ErrUnknownAddressType) {
return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
- github.com/btcsuite/btcd pinned to hash 306aecffea32
- github.com/btcsuite/btcutil pinned to 4c204d697803
- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
}
if err != nil {
return nil, nil, err
}
amount := 0.01
changeAddr := addr // TODO: fix this? or maybe its fine?
tx, err := baseTx(client, amount, changeAddr)
if err != nil {
return nil, nil, err
}
st, stPB, err := makeStream(path)
if err != nil {
return nil, nil, err
}
stPB.Author = details.Author
stPB.ReleaseTime = details.ReleaseTime
claim := &pb.Claim{
Title: details.Title,
Description: details.Description,
Type: &pb.Claim_Stream{Stream: stPB},
}
err = addClaimToTx(tx, claim, name, amount, addr)
if err != nil {
return nil, nil, err
}
// sign and send
signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
if err != nil {
return nil, nil, err
}
if !allInputsSigned {
return nil, nil, errors.Err("not all inputs for the tx could be signed")
}
err = reflect(st, reflectorAddress)
if err != nil {
return nil, nil, err
}
txid, err := client.SendRawTransaction(signedTx, false)
if err != nil {
return nil, nil, err
}
return signedTx, txid, nil
}
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
inputs, total, err := coinChooser(client, amount+txFee)
if err != nil {
return nil, err
}
change := total - amount - txFee
// create base raw tx
addresses := make(map[btcutil.Address]btcutil.Amount)
//changeAddr, err := client.GetNewAddress("")
changeAmount, err := btcutil.NewAmount(change)
if err != nil {
return nil, err
}
addresses[changeAddress] = changeAmount
lockTime := int64(0)
return client.CreateRawTransaction(inputs, addresses, &lockTime)
}
func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
utxos, err := client.ListUnspentMin(1)
if err != nil {
return nil, 0, err
}
sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
var utxo btcjson.ListUnspentResult
for _, u := range utxos {
if u.Spendable && u.Amount >= amount {
utxo = u
break
}
}
if utxo.TxID == "" {
return nil, 0, errors.Err("not enough utxos to create tx")
}
return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
}
func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
claimBytes, err := proto.Marshal(claim)
if err != nil {
return err
}
claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
amt, err := btcutil.NewAmount(amount)
if err != nil {
return err
}
script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
if err != nil {
return err
}
tx.AddTxOut(wire.NewTxOut(int64(amt), script))
return nil
}
func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
if err := tx.Serialize(buf); err != nil {
return "", errors.Err(err)
}
//txHex := hex.EncodeToString(buf.Bytes())
//spew.Dump(txHex)
decoded, err := client.DecodeRawTransaction(buf.Bytes())
if err != nil {
return "", err
}
data, err := json.MarshalIndent(decoded, "", " ")
return string(data), err
}
func reflect(st stream.Stream, reflectorAddress string) error {
// upload blobs to reflector
c := reflector.Client{}
err := c.Connect(reflectorAddress)
if err != nil {
return errors.Err(err)
}
for i, b := range st {
if i == 0 {
err = c.SendSDBlob(b)
} else {
err = c.SendBlob(b)
}
if err != nil {
return errors.Err(err)
}
}
return nil
}
func makeStream(path string) (stream.Stream, *pb.Stream, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, errors.Err(err)
}
defer file.Close()
enc := stream.NewEncoder(file)
s, err := enc.Stream()
if err != nil {
return nil, nil, errors.Err(err)
}
streamProto := &pb.Stream{
Source: &pb.Source{
SdHash: enc.SDBlob().Hash(),
Name: filepath.Base(file.Name()),
Size: uint64(enc.SourceLen()),
Hash: enc.SourceHash(),
},
}
mimeType, category := guessMimeType(filepath.Ext(file.Name()))
streamProto.Source.MediaType = mimeType
switch category {
case "video":
//t, err := streamVideoMetadata(path)
//if err != nil {
// return nil, nil, err
//}
streamProto.Type = &pb.Stream_Video{}
case "audio":
streamProto.Type = &pb.Stream_Audio{}
case "image":
streamProto.Type = &pb.Stream_Image{}
}
return s, streamProto, nil
}
func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
pkscript, err := txscript.PayToAddrScript(address)
if err != nil {
return nil, errors.Err(err)
}
return txscript.NewScriptBuilder().
AddOp(txscript.OP_NOP6). //OP_CLAIM_NAME
AddData([]byte(name)). //<name>
AddData(value). //<value>
AddOp(txscript.OP_2DROP). //OP_2DROP
AddOp(txscript.OP_DROP). //OP_DROP
AddOps(pkscript). //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
Script()
}
//func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
// mi, err := mediainfo.GetMediaInfo(path)
// if err != nil {
// return nil, err
// }
// return &pb.Stream_Video{
// Video: &pb.Video{
// Duration: uint32(mi.General.Duration / 1000),
// Height: uint32(mi.Video.Height),
// Width: uint32(mi.Video.Width),
// },
// }, nil
//}
+59
View File
@@ -0,0 +1,59 @@
package publish
import (
"encoding/json"
"io"
)
func LoadWallet(r io.Reader) (WalletFile, error) {
var w WalletFile
err := json.NewDecoder(r).Decode(&w)
return w, err
}
type WalletFile struct {
Name string `json:"name"`
Version int `json:"version"`
Preferences WalletPrefs `json:"preferences"`
Accounts []Account `json:"accounts"`
}
type Account struct {
AddressGenerator AddressGenerator `json:"address_generator"`
Certificates map[string]string `json:"certificates"`
Encrypted bool `json:"encrypted"`
Ledger string `json:"ledger"`
ModifiedOn float64 `json:"modified_on"`
Name string `json:"name"`
PrivateKey string `json:"private_key"`
PublicKey string `json:"public_key"`
Seed string `json:"seed"`
}
type AddressGenerator struct {
Name string `json:"name"`
Change AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
Receiving AddressGenParams `json:"receiving"`
}
type AddressGenParams struct {
Gap int `json:"gap"`
MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
}
type WalletPrefs struct {
Shared struct {
Ts float64 `json:"ts"`
Value struct {
Type string `json:"type"`
Value struct {
AppWelcomeVersion int `json:"app_welcome_version"`
Blocked []interface{} `json:"blocked"`
Sharing3P bool `json:"sharing_3P"`
Subscriptions []string `json:"subscriptions"`
Tags []string `json:"tags"`
} `json:"value"`
Version string `json:"version"`
} `json:"value"`
} `json:"shared"`
}
+93 -9
View File
@@ -1,23 +1,108 @@
# Reflector
A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO).
This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
Reflector is a central piece of software that providers LBRY with the following features:
- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
## Installation
coming soon
- Install mysql 8 (5.7 might work too)
- add a reflector user and database with password `reflector` with localhost access only
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/ittt/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
```bash
./prism-bin reflector \
--conf="none" \
--disable-uploads=true \
--use-db=false \
--upstream-reflector="reflector.lbry.com" \
--upstream-protocol="http" \
--request-queue-size=200 \
--disk-cache="2GB:/path/to/your/storage/:localdb" \
```
Create a systemd script if you want to run it automatically on startup or as a service.
## Usage
coming soon
Usage as reflector/blobcache:
```bash
Run reflector server
Usage:
prism reflector [flags]
Flags:
--disable-blocklist Disable blocklist watching/updating
--disable-uploads Disable uploads to this reflector server
--disk-cache string Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
-h, --help help for reflector
--http-peer-port int The port reflector will distribute content from over HTTP protocol (default 5569)
--http3-peer-port int The port reflector will distribute content from over HTTP3 protocol (default 5568)
--mem-cache int enable in-memory cache with a max size of this many blobs
--metrics-port int The port reflector will use for prometheus metrics (default 2112)
--optional-disk-cache string Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
--origin-endpoint string HTTP edge endpoint for standard HTTP retrieval
--origin-endpoint-fallback string HTTP edge endpoint for standard HTTP retrieval if first origin fails
--receiver-port int The port reflector will receive content from (default 5566)
--request-queue-size int How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
--tcp-peer-port int The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
--upstream-protocol string protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
--upstream-reflector string host:port of a reflector server where blobs are fetched from
--use-db Whether to connect to the reflector db or not (default true)
Global Flags:
--conf string Path to config. Use 'none' to disable (default "config.json")
-v, --verbose strings Verbose logging for specific components
```
Other uses:
```bash
Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
Usage:
prism [command]
Available Commands:
check-integrity check blobs integrity for a given path
cluster Start(join) to or Start a new cluster
decode Decode a claim value
dht Run dht node
getstream Get a stream from a reflector server
help Help about any command
peer Run peer server
populate-db populate local database with blobs from a disk storage
publish Publish a file
reflector Run reflector server
resolve Resolve a URL
send Send a file to a reflector
sendblob Send a random blob to a reflector server
start Runs full prism application with cluster, dht, peer server, and reflector server.
test Test things
upload Upload blobs to S3
version Print the version
Flags:
--conf string Path to config. Use 'none' to disable (default "config.json")
-h, --help help for prism
-v, --verbose strings Verbose logging for specific components
```
## Running from Source
This project requires [Go v1.10](https://golang.org/doc/install) or higher.
This project requires [Go v1.16](https://golang.org/doc/install).
On Ubuntu you can install it with `sudo snap install go --classic`
```
go get -u github.com/lbryio/reflector.go
cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
git clone git@github.com:lbryio/reflector.go.git
cd reflector.go
make
./bin/prism-bin
```
@@ -36,5 +121,4 @@ We take security seriously. Please contact security@lbry.com regarding any secur
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## Contact
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.com)
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)
+52 -66
View File
@@ -8,42 +8,49 @@ import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/reflector.go/wallet"
"github.com/lbryio/lbry.go/extras/errors"
types1 "github.com/lbryio/types/v1/go"
types2 "github.com/lbryio/types/v2/go"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/golang/protobuf/proto"
log "github.com/sirupsen/logrus"
)
const blocklistURL = "https://api.lbry.io/file/list_blocked"
const blocklistURL = "https://api.lbry.com/file/list_blocked"
func (s *Server) enableBlocklist(b store.Blocklister) {
// TODO: updateBlocklist should be killed when server is shutting down
updateBlocklist(b)
walletServers := []string{
"spv25.lbry.com:50001",
"spv26.lbry.com:50001",
"spv19.lbry.com:50001",
"spv14.lbry.com:50001",
}
updateBlocklist(b, walletServers, s.grp.Ch())
t := time.NewTicker(12 * time.Hour)
for {
select {
case <-s.grp.Ch():
return
case <-t.C:
updateBlocklist(b)
updateBlocklist(b, walletServers, s.grp.Ch())
}
}
}
func updateBlocklist(b store.Blocklister) {
values, err := blockedSdHashes()
func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) {
log.Debugf("blocklist update starting")
values, err := blockedSdHashes(walletServers, stopper)
if err != nil {
log.Error(err)
return
}
for _, v := range values {
for name, v := range values {
if v.Err != nil {
log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err)))
continue
}
@@ -52,17 +59,19 @@ func updateBlocklist(b store.Blocklister) {
log.Error(err)
}
}
log.Debugf("blocklist update done")
}
func blockedSdHashes() (map[string]valOrErr, error) {
resp, err := http.Get(blocklistURL)
func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) {
client := http.Client{Timeout: 1 * time.Second}
resp, err := client.Get(blocklistURL)
if err != nil {
return nil, errors.Err(err)
}
defer func() {
err := resp.Body.Close()
if err != nil {
log.Errorln(err)
log.Errorln(errors.Err(err))
}
}()
@@ -82,7 +91,7 @@ func blockedSdHashes() (map[string]valOrErr, error) {
return nil, errors.Prefix("list_blocked API call", r.Error)
}
return sdHashesForOutpoints(r.Data.Outpoints)
return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper)
}
type valOrErr struct {
@@ -91,20 +100,34 @@ type valOrErr struct {
}
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) {
values := make(map[string]valOrErr)
node := wallet.NewNode()
defer node.Shutdown()
err := node.Connect([]string{
"lbryumx1.lbry.io:50001",
"lbryumx2.lbry.io:50001",
}, nil)
err := node.Connect(walletServers, nil)
if err != nil {
return nil, err
return nil, errors.Err(err)
}
done := make(chan bool)
metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
select {
case <-done:
case <-stopper:
}
node.Shutdown()
}()
OutpointLoop:
for _, outpoint := range outpoints {
select {
case <-stopper:
break OutpointLoop
default:
}
parts := strings.Split(outpoint, ":")
if len(parts) != 2 {
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
@@ -117,57 +140,20 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
continue
}
resp, err := node.GetClaimsInTx(parts[0])
claim, err := node.GetClaimInTx(parts[0], nout)
if err != nil {
values[outpoint] = valOrErr{Err: err}
continue
}
var value []byte
for _, tx := range resp.Result {
if tx.Nout != nout {
continue
}
hash := hex.EncodeToString(claim.GetStream().GetSource().GetSdHash())
values[outpoint] = valOrErr{Value: hash, Err: nil}
}
value, err = hex.DecodeString(tx.Value)
break
}
if err != nil {
values[outpoint] = valOrErr{Err: err}
continue
} else if value == nil {
values[outpoint] = valOrErr{Err: errors.Err("outpoint not found")}
continue
}
hash, err := hashFromClaim(value)
values[outpoint] = valOrErr{Value: hash, Err: err}
select {
case done <- true:
default: // in case of race where stopper got stopped right after loop finished
}
return values, nil
}
func hashFromClaim(value []byte) (string, error) {
claim := &types1.Claim{}
err := proto.Unmarshal(value, claim)
if err != nil {
return "", err
}
if claim.GetStream().GetSource().GetSourceType() == types1.Source_lbry_sd_hash && claim.GetStream().GetSource().GetSource() != nil {
return hex.EncodeToString(claim.GetStream().GetSource().GetSource()), nil
}
claim2 := &types2.Claim{}
err = proto.Unmarshal(value, claim2)
if err != nil {
return "", err
}
stream, ok := claim2.GetType().(*types2.Claim_Stream)
if !ok || stream == nil {
return "", errors.Err("not a stream claim")
}
return hex.EncodeToString(claim2.GetStream().GetSource().GetSdHash()), nil
}
+61 -26
View File
@@ -2,12 +2,11 @@ package reflector
import (
"encoding/json"
"log"
"net"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/stream"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
@@ -36,8 +35,18 @@ func (c *Client) Close() error {
return c.conn.Close()
}
// SendBlob sends a send blob request to the client.
// SendBlob sends a blob to the server.
func (c *Client) SendBlob(blob stream.Blob) error {
return c.sendBlob(blob, false)
}
// SendSDBlob sends an SD blob request to the server.
func (c *Client) SendSDBlob(blob stream.Blob) error {
return c.sendBlob(blob, true)
}
// sendBlob does the actual blob sending
func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
if !c.connected {
return errors.Err("not connected")
}
@@ -47,10 +56,15 @@ func (c *Client) SendBlob(blob stream.Blob) error {
}
blobHash := blob.HashHex()
sendRequest, err := json.Marshal(sendBlobRequest{
BlobSize: blob.Size(),
BlobHash: blobHash,
})
var req sendBlobRequest
if isSDBlob {
req.SdBlobSize = blob.Size()
req.SdBlobHash = blobHash
} else {
req.BlobSize = blob.Size()
req.BlobHash = blobHash
}
sendRequest, err := json.Marshal(req)
if err != nil {
return err
}
@@ -62,30 +76,51 @@ func (c *Client) SendBlob(blob stream.Blob) error {
dec := json.NewDecoder(c.conn)
var sendResp sendBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
if isSDBlob {
var sendResp sendSdBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
}
if !sendResp.SendSdBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending SD blob " + blobHash[:8])
} else {
var sendResp sendBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
}
if !sendResp.SendBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending blob " + blobHash[:8])
}
if !sendResp.SendBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending blob " + blobHash[:8])
_, err = c.conn.Write(blob)
if err != nil {
return err
}
var transferResp blobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedBlob {
return errors.Err("server did not received blob")
if isSDBlob {
var transferResp sdBlobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedSdBlob {
return errors.Err("server did not received SD blob")
}
} else {
var transferResp blobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedBlob {
return errors.Err("server did not received blob")
}
}
return nil
+38 -32
View File
@@ -8,14 +8,14 @@ import (
"io"
"io/ioutil"
"net"
"strconv"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/stream"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
@@ -32,33 +32,32 @@ const (
maxBlobSize = stream.MaxBlobSize
)
var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
// Server is and instance of the reflector server. It houses the blob store and listener.
type Server struct {
Timeout time.Duration // timeout to read or write next message
StatLogger *log.Logger // logger to log stats
StatReportFrequency time.Duration // how often to log stats
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
store store.BlobStore
grp *stop.Group
stats *Stats
underlyingStore store.BlobStore
outerStore store.BlobStore
grp *stop.Group
}
// NewServer returns an initialized reflector server pointer.
func NewServer(store store.BlobStore) *Server {
func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
return &Server{
Timeout: DefaultTimeout,
store: store,
grp: stop.New(),
Timeout: DefaultTimeout,
underlyingStore: underlying,
outerStore: outer,
grp: stop.New(),
}
}
// Shutdown shuts down the reflector server gracefully.
func (s *Server) Shutdown() {
log.Println("shutting down reflector server...")
s.stats.Shutdown()
s.grp.StopAndWait()
log.Println("reflector server stopped")
}
@@ -70,9 +69,10 @@ func (s *Server) Start(address string) error {
return errors.Err(err)
}
log.Println("reflector listening on " + address)
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
<-s.grp.Ch()
err := l.Close()
if err != nil {
@@ -82,20 +82,19 @@ func (s *Server) Start(address string) error {
}()
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
s.listenAndServe(l)
s.grp.Done()
}()
s.stats = NewStatLogger("UPLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
if s.StatLogger != nil && s.StatReportFrequency > 0 {
s.stats.Start()
}
if s.EnableBlocklist {
if b, ok := s.store.(store.Blocklister); ok {
if b, ok := s.underlyingStore.(store.Blocklister); ok {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
s.enableBlocklist(b)
s.grp.Done()
}()
@@ -118,7 +117,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
log.Error(err)
} else {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
s.handleConn(conn)
s.grp.Done()
}()
@@ -133,7 +134,9 @@ func (s *Server) handleConn(conn net.Conn) {
close(connNeedsClosing)
}()
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
defer s.grp.Done()
select {
case <-connNeedsClosing:
@@ -173,7 +176,10 @@ func (s *Server) handleConn(conn net.Conn) {
}
func (s *Server) doError(conn net.Conn, err error) error {
shouldLog := s.stats.AddError(err)
if err == nil {
return nil
}
shouldLog := metrics.TrackError(metrics.DirectionUpload, err)
if shouldLog {
log.Errorln(errors.FullTrace(err))
}
@@ -195,13 +201,13 @@ func (s *Server) receiveBlob(conn net.Conn) error {
}
var wantsBlob bool
if bl, ok := s.store.(store.Blocklister); ok {
if bl, ok := s.underlyingStore.(store.Blocklister); ok {
wantsBlob, err = bl.Wants(blobHash)
if err != nil {
return err
}
} else {
blobExists, err := s.store.Has(blobHash)
blobExists, err := s.underlyingStore.Has(blobHash)
if err != nil {
return err
}
@@ -211,7 +217,7 @@ func (s *Server) receiveBlob(conn net.Conn) error {
var neededBlobs []string
if isSdBlob && !wantsBlob {
if nbc, ok := s.store.(neededBlobChecker); ok {
if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
if err != nil {
return err
@@ -254,17 +260,17 @@ func (s *Server) receiveBlob(conn net.Conn) error {
log.Debugln("Got blob " + blobHash[:8])
if isSdBlob {
err = s.store.PutSD(blobHash, blob)
err = s.outerStore.PutSD(blobHash, blob)
} else {
err = s.store.Put(blobHash, blob)
err = s.outerStore.Put(blobHash, blob)
}
if err != nil {
return err
}
s.stats.AddBlob()
metrics.MtrInBytesReflector.Add(float64(len(blob)))
metrics.BlobUploadCount.Inc()
if isSdBlob {
s.stats.AddStream()
metrics.SDBlobUploadCount.Inc()
}
return s.sendTransferResponse(conn, true, isSdBlob)
}
@@ -311,7 +317,7 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
return blobSize, blobHash, isSdBlob, errors.Err("blob hash is empty")
}
if blobSize > maxBlobSize {
return blobSize, blobHash, isSdBlob, errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
return blobSize, blobHash, isSdBlob, errors.Err(ErrBlobTooBig)
}
if blobSize == 0 {
return blobSize, blobHash, isSdBlob, errors.Err("0-byte blob received")
+7 -6
View File
@@ -9,9 +9,10 @@ import (
"testing"
"time"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/davecgh/go-spew/spew"
"github.com/phayes/freeport"
)
@@ -22,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
t.Fatal(err)
}
srv := NewServer(&store.MemoryBlobStore{})
srv := NewServer(store.NewMemStore(), store.NewMemStore())
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
@@ -119,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
t.Fatal(err)
}
srv := NewServer(&store.MemoryBlobStore{})
srv := NewServer(store.NewMemStore(), store.NewMemStore())
srv.Timeout = testTimeout
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
@@ -161,7 +162,7 @@ func TestServer_Timeout(t *testing.T) {
//}
type mockPartialStore struct {
store.MemoryBlobStore
*store.MemStore
missing []string
}
@@ -181,7 +182,7 @@ func TestServer_PartialUpload(t *testing.T) {
missing[i] = bits.Rand().String()
}
st := store.BlobStore(&mockPartialStore{missing: missing})
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
if _, ok := st.(neededBlobChecker); !ok {
t.Fatal("mock does not implement the relevant interface")
}
@@ -190,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
t.Fatal(err)
}
srv := NewServer(st)
srv := NewServer(st, st)
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
-122
View File
@@ -1,122 +0,0 @@
package reflector
import (
"fmt"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
// TODO: store daily stats too. and maybe other intervals
type Stats struct {
mu *sync.Mutex
blobs int
streams int
errors map[string]int
started bool
name string
logger *log.Logger
logFreq time.Duration
grp *stop.Group
}
func NewStatLogger(name string, logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *Stats {
return &Stats{
mu: &sync.Mutex{},
grp: stop.New(parentGrp),
logger: logger,
logFreq: logFreq,
errors: make(map[string]int),
name: name,
}
}
func (s *Stats) Start() {
s.started = true
s.grp.Add(1)
go func() {
defer s.grp.Done()
s.runSlackLogger()
}()
}
func (s *Stats) Shutdown() {
if !s.started {
return
}
s.log()
s.grp.StopAndWait()
s.started = false
}
func (s *Stats) AddBlob() {
s.mu.Lock()
defer s.mu.Unlock()
s.blobs++
}
func (s *Stats) AddStream() {
s.mu.Lock()
defer s.mu.Unlock()
s.streams++
}
func (s *Stats) AddError(e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
return
}
err := errors.Wrap(e, 0)
name := err.TypeName()
if strings.Contains(err.Error(), "i/o timeout") { // hit a read or write deadline
name = "i/o timeout"
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
name = "read conn reset"
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
name = "unexpected EOF"
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
name = "write broken pipe"
} else {
shouldLog = true
}
s.mu.Lock()
defer s.mu.Unlock()
s.errors[name]++
return
}
func (s *Stats) runSlackLogger() {
t := time.NewTicker(s.logFreq)
for {
select {
case <-s.grp.Ch():
return
case <-t.C:
s.log()
}
}
}
func (s *Stats) log() {
s.mu.Lock()
blobs, streams := s.blobs, s.streams
s.blobs, s.streams = 0, 0
errStr := ""
for name, count := range s.errors {
errStr += fmt.Sprintf("%d %s, ", count, name)
delete(s.errors, name)
}
s.mu.Unlock()
if len(errStr) > 2 {
errStr = errStr[:len(errStr)-2] // trim last comma and space
}
s.logger.Printf("%s stats: %d blobs, %d streams, errors: %s", s.name, blobs, streams, errStr)
}
+58 -43
View File
@@ -8,10 +8,11 @@ import (
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
)
@@ -24,27 +25,31 @@ const (
errInc
)
type Uploader struct {
db *db.SQL
store *store.DBBackedS3Store // could just be store.BlobStore interface
workers int
skipExistsCheck bool
stopper *stop.Group
countChan chan increment
count struct {
total, alreadyStored, sd, blob, err int
}
type Summary struct {
Total, AlreadyStored, Sd, Blob, Err int
}
func NewUploader(db *db.SQL, store *store.DBBackedS3Store, workers int, skipExistsCheck bool) *Uploader {
type Uploader struct {
db *db.SQL
store *store.DBBackedStore // could just be store.BlobStore interface
workers int
skipExistsCheck bool
deleteBlobsAfterUpload bool
stopper *stop.Group
countChan chan increment
count Summary
}
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
return &Uploader{
db: db,
store: store,
workers: workers,
skipExistsCheck: skipExistsCheck,
stopper: stop.New(),
countChan: make(chan increment),
db: db,
store: store,
workers: workers,
skipExistsCheck: skipExistsCheck,
deleteBlobsAfterUpload: deleteBlobsAfterUpload,
stopper: stop.New(),
countChan: make(chan increment),
}
}
@@ -59,32 +64,34 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
return err
}
u.count.total = len(paths)
u.count.Total = len(paths)
hashes := make([]string, len(paths))
for i, p := range paths {
hashes[i] = path.Base(p)
}
log.Infoln("checking for existing blobs")
log.Debug("checking for existing blobs")
var exists map[string]bool
if !u.skipExistsCheck {
exists, err = u.db.HasBlobs(hashes)
exists, err = u.db.HasBlobs(hashes, false)
if err != nil {
return err
}
u.count.alreadyStored = len(exists)
u.count.AlreadyStored = len(exists)
}
log.Infof("%d new blobs to upload", u.count.total-u.count.alreadyStored)
log.Debugf("%d new blobs to upload", u.count.Total-u.count.AlreadyStored)
workerWG := sync.WaitGroup{}
pathChan := make(chan string)
for i := 0; i < u.workers; i++ {
workerWG.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
go func(i int) {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
defer workerWG.Done()
defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
u.worker(pathChan)
@@ -93,7 +100,9 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
countWG := sync.WaitGroup{}
countWG.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
defer countWG.Done()
u.counter()
}()
@@ -117,17 +126,14 @@ Upload:
countWG.Wait()
u.stopper.Stop()
log.Infoln("SUMMARY")
log.Infof("%d blobs total", u.count.total)
log.Infof("%d blobs already stored", u.count.alreadyStored)
log.Infof("%d SD blobs uploaded", u.count.sd)
log.Infof("%d content blobs uploaded", u.count.blob)
log.Infof("%d errors encountered", u.count.err)
log.Debugf(
"upload stats: %d blobs total, %d already stored, %d SD blobs uploaded, %d content blobs uploaded, %d errors",
u.count.Total, u.count.AlreadyStored, u.count.Sd, u.count.Blob, u.count.Err,
)
return nil
}
// worker reads paths from a channel and uploads them
// worker reads paths from a channel, uploads them, and optionally deletes them
func (u *Uploader) worker(pathChan chan string) {
for {
select {
@@ -141,6 +147,11 @@ func (u *Uploader) worker(pathChan chan string) {
err := u.uploadBlob(filepath)
if err != nil {
log.Errorln(err)
} else if u.deleteBlobsAfterUpload {
err = os.Remove(filepath)
if err != nil {
log.Errorln(errors.Prefix("deleting blob", err))
}
}
}
}
@@ -156,7 +167,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
blob, err := ioutil.ReadFile(filepath)
if err != nil {
return err
return errors.Err(err)
}
hash := BlobHash(blob)
@@ -165,17 +176,17 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
if IsValidJSON(blob) {
log.Debugf("Uploading SD blob %s", hash)
log.Debugf("uploading SD blob %s", hash)
err := u.store.PutSD(hash, blob)
if err != nil {
return errors.Prefix("Uploading SD blob "+hash, err)
return errors.Prefix("uploading SD blob "+hash, err)
}
u.inc(sdInc)
} else {
log.Debugf("Uploading blob %s", hash)
log.Debugf("uploading blob %s", hash)
err = u.store.Put(hash, blob)
if err != nil {
return errors.Prefix("Uploading blob "+hash, err)
return errors.Prefix("uploading blob "+hash, err)
}
u.inc(blobInc)
}
@@ -198,19 +209,23 @@ func (u *Uploader) counter() {
}
switch incrementType {
case sdInc:
u.count.sd++
u.count.Sd++
case blobInc:
u.count.blob++
u.count.Blob++
case errInc:
u.count.err++
u.count.Err++
}
}
if (u.count.sd+u.count.blob)%50 == 0 {
log.Infof("%d of %d done (%s elapsed, %.3fs per blob)", u.count.sd+u.count.blob, u.count.total-u.count.alreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.sd+u.count.blob))
if (u.count.Sd+u.count.Blob)%50 == 0 {
log.Debugf("%d of %d done (%s elapsed, %.3fs per blob)", u.count.Sd+u.count.Blob, u.count.Total-u.count.AlreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.Sd+u.count.Blob))
}
}
}
func (u *Uploader) GetSummary() Summary {
return u.count
}
func (u *Uploader) inc(t increment) {
select {
case u.countChan <- t:
+92
View File
@@ -0,0 +1,92 @@
package http
import (
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/gin-gonic/gin"
)
func (s *Server) getBlob(c *gin.Context) {
waiter := &sync.WaitGroup{}
waiter.Add(1)
enqueue(&blobRequest{c: c, finished: waiter})
waiter.Wait()
}
func (s *Server) HandleGetBlob(c *gin.Context) {
start := time.Now()
hash := c.Query("hash")
if s.missesCache.Has(hash) {
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
c.Header("Via", serialized)
if err != nil {
_ = c.Error(errors.Err(err))
c.String(http.StatusInternalServerError, err.Error())
return
}
c.AbortWithStatus(http.StatusNotFound)
return
}
blob, trace, err := s.store.Get(hash)
if err != nil {
serialized, serializeErr := trace.Serialize()
if serializeErr != nil {
_ = c.Error(errors.Prefix(serializeErr.Error(), err))
c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
return
}
c.Header("Via", serialized)
if errors.Is(err, store.ErrBlobNotFound) {
_ = s.missesCache.Set(hash, true)
c.AbortWithStatus(http.StatusNotFound)
return
}
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
return
}
serialized, err := trace.Serialize()
if err != nil {
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
return
}
metrics.MtrOutBytesHttp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.HttpDownloadCount.Inc()
c.Header("Via", serialized)
c.Header("Content-Disposition", "filename="+hash)
c.Data(http.StatusOK, "application/octet-stream", blob)
}
func (s *Server) hasBlob(c *gin.Context) {
hash := c.Query("hash")
has, err := s.store.Has(hash)
if err != nil {
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
return
}
if has {
c.Status(http.StatusNoContent)
return
}
c.Status(http.StatusNotFound)
}
func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
c.JSON(500, gin.H{
"title": "Error",
"err": err,
})
}
+79
View File
@@ -0,0 +1,79 @@
package http
import (
"context"
"net/http"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/bluele/gcache"
nice "github.com/ekyoung/gin-nice-recovery"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
concurrentRequests int
missesCache gcache.Cache
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
}
}
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down HTTP server")
s.grp.StopAndWait()
log.Debug("HTTP server stopped")
}
// Start starts the server listener to handle connections.
func (s *Server) Start(address string) error {
gin.SetMode(gin.ReleaseMode)
router := gin.Default()
router.GET("/blob", s.getBlob)
router.HEAD("/blob", s.hasBlob)
// Install nice.Recovery, passing the handler to call after recovery
router.Use(nice.Recovery(s.recoveryHandler))
srv := &http.Server{
Addr: address,
Handler: router,
}
go s.listenForShutdown(srv)
go InitWorkers(s, s.concurrentRequests)
// Initializing the server in a goroutine so that
// it won't block the graceful shutdown handling below
s.grp.Add(1)
go func() {
defer s.grp.Done()
log.Println("HTTP server listening on " + address)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
}()
return nil
}
func (s *Server) listenForShutdown(listener *http.Server) {
<-s.grp.Ch()
// The context is used to inform the server it has 5 seconds to finish
// the request it is currently handling
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := listener.Shutdown(ctx); err != nil {
log.Fatal("Server forced to shutdown:", err)
}
}
+46
View File
@@ -0,0 +1,46 @@
package http
import (
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gin-gonic/gin"
)
type blobRequest struct {
c *gin.Context
finished *sync.WaitGroup
}
var getReqCh = make(chan *blobRequest, 20000)
func InitWorkers(server *Server, workers int) {
stopper := stop.New(server.grp)
for i := 0; i < workers; i++ {
metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
go func(worker int) {
defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
for {
select {
case <-stopper.Ch():
case r := <-getReqCh:
process(server, r)
metrics.HttpBlobReqQueue.Dec()
}
}
}(i)
}
}
func enqueue(b *blobRequest) {
metrics.HttpBlobReqQueue.Inc()
getReqCh <- b
}
func process(server *Server, r *blobRequest) {
server.HandleGetBlob(r.c)
r.finished.Done()
}
+141
View File
@@ -0,0 +1,141 @@
package http3
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
Timeout time.Duration
conn *http.Client
roundTripper *http3.RoundTripper
ServerAddr string
}
// Close closes the connection with the client.
func (c *Client) Close() error {
c.conn.CloseIdleConnections()
return c.roundTripper.Close()
}
// GetStream gets a stream
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
var sd stream.SDBlob
b, _, err := c.GetBlob(sdHash)
if err != nil {
return nil, err
}
err = sd.FromBlob(b)
if err != nil {
return nil, err
}
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
s[0] = b
for i := 0; i < len(sd.BlobInfos)-1; i++ {
var trace shared.BlobTrace
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
if err != nil {
return nil, err
}
log.Debug(trace.String())
}
return s, nil
}
// HasBlob checks if the blob is available
func (c *Client) HasBlob(hash string) (bool, error) {
resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
if err != nil {
return false, errors.Err(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
}
// GetBlob gets a blob
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
} else if resp.StatusCode != http.StatusOK {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
}
tmp := getBuffer()
defer putBuffer(tmp)
serialized := resp.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), "http3")
if serialized != "" {
parsedTrace, err := shared.Deserialize(serialized)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
trace = *parsedTrace
}
written, err := io.Copy(tmp, resp.Body)
if err != nil {
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
}
blob := make([]byte, written)
copy(blob, tmp.Bytes())
metrics.MtrInBytesUdp.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), "http3"), nil
}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
+214
View File
@@ -0,0 +1,214 @@
package http3
import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"net/http"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gorilla/mux"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
concurrentRequests int
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
}
}
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down http3 peer server")
s.grp.StopAndWait()
log.Debug("http3 peer server stopped")
}
func (s *Server) logError(e error) {
if e == nil {
return
}
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
if shouldLog {
log.Errorln(errors.FullTrace(e))
}
}
type availabilityResponse struct {
LbrycrdAddress string `json:"lbrycrd_address"`
IsAvailable bool `json:"is_available"`
}
// Start starts the server listener to handle connections.
func (s *Server) Start(address string) error {
log.Println("HTTP3 peer listening on " + address)
window500M := 500 * 1 << 20
quicConf := &quic.Config{
MaxStreamReceiveWindow: uint64(window500M),
MaxConnectionReceiveWindow: uint64(window500M),
EnableDatagrams: true,
HandshakeIdleTimeout: 4 * time.Second,
MaxIdleTimeout: 20 * time.Second,
}
r := mux.NewRouter()
r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
waiter := &sync.WaitGroup{}
waiter.Add(1)
enqueue(&blobRequest{request: r, reply: w, finished: waiter})
waiter.Wait()
})
r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
requestedBlob := vars["hash"]
blobExists, err := s.store.Has(requestedBlob)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
s.logError(err)
return
}
if !blobExists {
w.WriteHeader(http.StatusNotFound)
}
// LbrycrdAddress to be used when paying for data. Not implemented yet.
const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
resp, err := json.Marshal(availabilityResponse{
LbrycrdAddress: LbrycrdAddress,
IsAvailable: blobExists,
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
s.logError(err)
return
}
_, err = w.Write(resp)
if err != nil {
s.logError(err)
}
})
server := http3.Server{
Server: &http.Server{
Handler: r,
Addr: address,
TLSConfig: generateTLSConfig(),
},
QuicConfig: quicConf,
}
go InitWorkers(s, s.concurrentRequests)
go s.listenForShutdown(&server)
s.grp.Add(1)
go func() {
s.listenAndServe(&server)
s.grp.Done()
}()
return nil
}
// Setup a bare-bones TLS config for the server
func generateTLSConfig() *tls.Config {
key, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
panic(err)
}
template := x509.Certificate{SerialNumber: big.NewInt(1)}
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
if err != nil {
panic(err)
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
panic(err)
}
return &tls.Config{
Certificates: []tls.Certificate{tlsCert},
NextProtos: []string{"http3-reflector-server"},
}
}
func (s *Server) listenAndServe(server *http3.Server) {
err := server.ListenAndServe()
if err != nil && err.Error() != "server closed" {
log.Errorln(errors.FullTrace(err))
}
}
func (s *Server) listenForShutdown(listener *http3.Server) {
<-s.grp.Ch()
err := listener.Close()
if err != nil {
log.Error("error closing listener for peer server - ", err)
}
}
func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
requestedBlob := vars["hash"]
traceParam := r.URL.Query().Get("trace")
var err error
wantsTrace := false
if traceParam != "" {
wantsTrace, err = strconv.ParseBool(traceParam)
if err != nil {
wantsTrace = false
}
}
blob, trace, err := s.store.Get(requestedBlob)
if wantsTrace {
serialized, err := trace.Serialize()
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
w.Header().Add("Via", serialized)
log.Debug(trace.String())
}
if err != nil {
if errors.Is(err, store.ErrBlobNotFound) {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
s.logError(err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
_, err = w.Write(blob)
if err != nil {
s.logError(err)
}
metrics.MtrOutBytesUdp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.Http3DownloadCount.Inc()
}
+117
View File
@@ -0,0 +1,117 @@
package http3
import (
"crypto/tls"
"crypto/x509"
"net/http"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
NotFoundCache *sync.Map
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts, NotFoundCache: &sync.Map{}}
}
func (p *Store) getClient() (*Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.RoundTripper{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QuicConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
c := &Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: p.opts.Address,
}
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "http3" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
}
}
c, err := p.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
p.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
return c.GetBlob(hash)
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *Store) Shutdown() {
}
+46
View File
@@ -0,0 +1,46 @@
package http3
import (
"net/http"
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/stop"
)
type blobRequest struct {
request *http.Request
reply http.ResponseWriter
finished *sync.WaitGroup
}
var getReqCh = make(chan *blobRequest, 20000)
func InitWorkers(server *Server, workers int) {
stopper := stop.New(server.grp)
for i := 0; i < workers; i++ {
metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
go func(worker int) {
defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
for {
select {
case <-stopper.Ch():
case r := <-getReqCh:
metrics.Http3BlobReqQueue.Dec()
process(server, r)
}
}
}(i)
}
}
func enqueue(b *blobRequest) {
metrics.Http3BlobReqQueue.Inc()
getReqCh <- b
}
func process(server *Server, r *blobRequest) {
server.HandleGetBlob(r.reply, r.request)
r.finished.Done()
}
+207
View File
@@ -0,0 +1,207 @@
package peer
import (
"bufio"
"encoding/hex"
"encoding/json"
"io"
"net"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
Timeout time.Duration
conn net.Conn
buf *bufio.Reader
connected bool
}
// Connect connects to a specific clients and errors if it cannot be contacted.
func (c *Client) Connect(address string) error {
var err error
if c.Timeout == 0 {
c.Timeout = 5 * time.Second
}
c.conn, err = net.Dial("tcp4", address)
if err != nil {
return err
}
c.connected = true
c.buf = bufio.NewReader(c.conn)
return nil
}
// Close closes the connection with the client.
func (c *Client) Close() error {
c.connected = false
return c.conn.Close()
}
// GetStream gets a stream
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
if !c.connected {
return nil, errors.Err("not connected")
}
var sd stream.SDBlob
b, trace, err := c.GetBlob(sdHash)
if err != nil {
return nil, err
}
log.Debug(trace.String())
err = sd.FromBlob(b)
if err != nil {
return nil, err
}
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
s[0] = b
for i := 0; i < len(sd.BlobInfos)-1; i++ {
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
if err != nil {
return nil, err
}
log.Debug(trace.String())
}
return s, nil
}
// HasBlob checks if the blob is available
func (c *Client) HasBlob(hash string) (bool, error) {
if !c.connected {
return false, errors.Err("not connected")
}
sendRequest, err := json.Marshal(availabilityRequest{
RequestedBlobs: []string{hash},
})
if err != nil {
return false, err
}
err = c.write(sendRequest)
if err != nil {
return false, err
}
var resp availabilityResponse
err = c.read(&resp)
if err != nil {
return false, err
}
for _, h := range resp.AvailableBlobs {
if h == hash {
return true, nil
}
}
return false, nil
}
// GetBlob gets a blob
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if !c.connected {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
}
sendRequest, err := json.Marshal(blobRequest{
RequestedBlob: hash,
})
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
}
err = c.write(sendRequest)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
}
var resp blobResponse
err = c.read(&resp)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
}
trace := shared.NewBlobTrace(time.Since(start), "tcp")
if resp.RequestTrace != nil {
trace = *resp.RequestTrace
}
if resp.IncomingBlob.Error != "" {
return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
}
if resp.IncomingBlob.BlobHash != hash {
return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
}
if resp.IncomingBlob.Length <= 0 {
return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
}
log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
blob, err := c.readRawBlob(resp.IncomingBlob.Length)
if err != nil {
return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
}
metrics.MtrInBytesTcp.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), "tcp"), nil
}
func (c *Client) read(v interface{}) error {
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
if err != nil {
return errors.Err(err)
}
m, err := readNextMessage(c.buf)
if err != nil {
return err
}
log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr())
err = json.Unmarshal(m, v)
return errors.Err(err)
}
func (c *Client) readRawBlob(blobSize int) ([]byte, error) {
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
if err != nil {
return nil, errors.Err(err)
}
blob := make([]byte, blobSize)
n, err := io.ReadFull(c.buf, blob)
log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr())
return blob, errors.Err(err)
}
func (c *Client) write(b []byte) error {
err := c.conn.SetWriteDeadline(time.Now().Add(c.Timeout))
if err != nil {
return errors.Err(err)
}
log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr())
n, err := c.conn.Write(b)
if err == nil && n != len(b) {
err = io.ErrShortWrite
}
return errors.Err(err)
}
+93 -101
View File
@@ -2,17 +2,22 @@ package peer
import (
"bufio"
"encoding/hex"
"encoding/json"
ee "errors"
"io"
"net"
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
@@ -26,28 +31,25 @@ const (
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
StatLogger *log.Logger // logger to log stats
StatReportFrequency time.Duration // how often to log stats
store store.BlobStore
disk *store.DiskStore
closed bool
grp *stop.Group
stats *reflector.Stats
grp *stop.Group
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore) *Server {
func NewServer(blobstore store.BlobStore) *Server {
return &Server{
store: store,
store: blobstore,
disk: store.NewDiskStore("/external/blobs", 2),
grp: stop.New(),
}
}
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down peer server...")
s.stats.Shutdown()
log.Debug("shutting down peer server")
s.grp.StopAndWait()
log.Debug("peer server stopped")
}
@@ -67,11 +69,6 @@ func (s *Server) Start(address string) error {
s.grp.Done()
}()
s.stats = reflector.NewStatLogger("DOWNLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
if s.StatLogger != nil && s.StatReportFrequency > 0 {
s.stats.Start()
}
return nil
}
@@ -94,7 +91,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
log.Error(errors.Prefix("accepting conn", err))
} else {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
s.handleConnection(conn)
s.grp.Done()
}()
@@ -110,6 +109,7 @@ func (s *Server) handleConnection(conn net.Conn) {
}()
timeoutDuration := 1 * time.Minute
buf := bufio.NewReader(conn)
for {
var request []byte
@@ -120,7 +120,7 @@ func (s *Server) handleConnection(conn net.Conn) {
log.Error(errors.FullTrace(err))
}
request, err = readNextRequest(conn)
request, err = readNextMessage(buf)
if err != nil {
if err != io.EOF {
s.logError(err)
@@ -133,9 +133,9 @@ func (s *Server) handleConnection(conn net.Conn) {
log.Error(errors.FullTrace(err))
}
response, err = s.handleCompositeRequest(request)
response, blobHash, err := s.handleCompositeRequest(request)
if err != nil {
log.Error(err)
log.Error(errors.FullTrace(err))
return
}
@@ -145,6 +145,7 @@ func (s *Server) handleConnection(conn net.Conn) {
}
n, err := conn.Write(response)
s.disk.Sendfile(blobHash, conn)
if err != nil {
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
s.logError(err)
@@ -166,14 +167,14 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
var request availabilityRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
return nil, errors.Err(err)
}
availableBlobs := []string{}
for _, blobHash := range request.RequestedBlobs {
exists, err := s.store.Has(blobHash)
if err != nil {
return []byte{}, err
return nil, err
}
if exists {
availableBlobs = append(availableBlobs, blobHash)
@@ -183,51 +184,55 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
}
func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
var request paymentRateRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
}
//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
// var request paymentRateRequest
// err := json.Unmarshal(data, &request)
// if err != nil {
// return nil, err
// }
//
// offerReply := paymentRateAccepted
// if request.BlobDataPaymentRate < 0 {
// offerReply = paymentRateTooLow
// }
//
// return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
//}
//
//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
// var request blobRequest
// err := json.Unmarshal(data, &request)
// if err != nil {
// return nil, err
// }
//
// log.Debugln("Sending blob " + request.RequestedBlob[:8])
//
// blob, err := s.store.Get(request.RequestedBlob)
// if err != nil {
// return nil, err
// }
//
// response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
// BlobHash: reflector.BlobHash(blob),
// Length: len(blob),
// }})
// if err != nil {
// return nil, err
// }
//
// return append(response, blob...), nil
//}
offerReply := paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
offerReply = paymentRateTooLow
}
return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
}
func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
var request blobRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
}
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, err := s.store.Get(request.RequestedBlob)
if err != nil {
return []byte{}, err
}
response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
BlobHash: reflector.BlobHash(blob),
Length: len(blob),
}})
if err != nil {
return []byte{}, err
}
return append(response, blob...), nil
}
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
func (s *Server) handleCompositeRequest(data []byte) ([]byte, string, error) {
var request compositeRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
var je *json.SyntaxError
if ee.As(err, &je) {
return nil, "", errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
}
return nil, "", errors.Err(err)
}
response := compositeResponse{
@@ -239,7 +244,7 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
for _, blobHash := range request.RequestedBlobs {
exists, err := s.store.Has(blobHash)
if err != nil {
return []byte{}, err
return nil, "", err
}
if exists {
availableBlobs = append(availableBlobs, blobHash)
@@ -255,70 +260,56 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
var blob []byte
if request.RequestedBlob != "" {
if len(request.RequestedBlob) != stream.BlobHashHexLength {
return nil, "", errors.Err("Invalid blob hash length")
}
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, err = s.store.Get(request.RequestedBlob)
if errors.Is(err, store.ErrBlobNotFound) {
size, err := s.disk.Size(request.RequestedBlob)
if err != nil {
response.IncomingBlob = incomingBlob{
Error: err.Error(),
}
} else if err != nil {
return []byte{}, err
} else {
response.IncomingBlob = incomingBlob{
BlobHash: reflector.BlobHash(blob),
Length: len(blob),
BlobHash: request.RequestedBlob,
Length: int(size),
}
s.stats.AddBlob()
}
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.PeerDownloadCount.Inc()
}
respData, err := json.Marshal(response)
if err != nil {
return []byte{}, err
return nil, "", err
}
return append(respData, blob...), nil
return respData, request.RequestedBlob, nil
}
func (s *Server) logError(e error) {
if e == nil {
return
}
shouldLog := s.stats.AddError(e)
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
if shouldLog {
log.Errorln(errors.FullTrace(e))
}
return
// old stuff below. its here for posterity, because we're gonna have to deal with these errors someday for real
//err := errors.Wrap(e, 0)
// these happen because the peer protocol does not have a way to cancel blob downloads
// so the client will just close the connection if its in the middle of downloading a blob
// but receives the blob from a different peer first or simply goes offline (timeout)
//if strings.Contains(err.Error(), "connection reset by peer") ||
// strings.Contains(err.Error(), "i/o timeout") ||
// strings.Contains(err.Error(), "broken pipe") {
// return
//}
//
//log.Error(errors.FullTrace(e))
}
func readNextRequest(conn net.Conn) ([]byte, error) {
request := make([]byte, 0)
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
msg := make([]byte, 0)
eof := false
buf := bufio.NewReader(conn)
for {
chunk, err := buf.ReadBytes('}')
if err != nil {
if err != io.EOF {
//log.Errorln("readBytes error:", err) // logged by caller
return request, err
return msg, err
}
eof = true
}
@@ -327,14 +318,14 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
//spew.Dump(chunk)
if len(chunk) > 0 {
request = append(request, chunk...)
msg = append(msg, chunk...)
if len(request) > maxRequestSize {
return request, errRequestTooLarge
if len(msg) > maxRequestSize {
return msg, errRequestTooLarge
}
// yes, this is how the peer protocol knows when the request finishes
if reflector.IsValidJSON(request) {
if reflector.IsValidJSON(msg) {
break
}
}
@@ -349,11 +340,11 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
// spew.Dump(request)
//}
if len(request) == 0 && eof {
return []byte{}, io.EOF
if len(msg) == 0 && eof {
return nil, io.EOF
}
return request, nil
return msg, nil
}
const (
@@ -395,6 +386,7 @@ type incomingBlob struct {
}
type blobResponse struct {
IncomingBlob incomingBlob `json:"incoming_blob"`
RequestTrace *shared.BlobTrace
}
type compositeRequest struct {
@@ -34,7 +34,7 @@ var availabilityRequests = []pair{
}
func getServer(t *testing.T, withBlobs bool) *Server {
st := store.MemoryBlobStore{}
st := store.NewMemStore()
if withBlobs {
for k, v := range blobs {
err := st.Put(k, v)
@@ -43,7 +43,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
}
}
}
return NewServer(&st)
return NewServer(st)
}
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
+82
View File
@@ -0,0 +1,82 @@
package peer
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts}
}
func (p *Store) getClient() (*Client, error) {
c := &Client{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "peer" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, store.ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *Store) Shutdown() {
}
+6
View File
@@ -0,0 +1,6 @@
package shared
import "github.com/lbryio/lbry.go/v2/extras/errors"
//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
var ErrNotImplemented = errors.Base("this store does not implement this method")
+82
View File
@@ -0,0 +1,82 @@
package shared
import (
"encoding/json"
"fmt"
"os"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
type BlobStack struct {
Timing time.Duration `json:"timing"`
OriginName string `json:"origin_name"`
HostName string `json:"host_name"`
}
type BlobTrace struct {
Stacks []BlobStack `json:"stacks"`
}
var hostName *string
func getHostName() string {
if hostName == nil {
hn, err := os.Hostname()
if err != nil {
hn = "unknown"
}
hostName = &hn
}
return *hostName
}
func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
b.Stacks = append(b.Stacks, BlobStack{
Timing: timing,
OriginName: originName,
HostName: getHostName(),
})
return *b
}
func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
b.Stacks = append(b.Stacks, otherTrance.Stacks...)
return *b
}
func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
b := BlobTrace{}
b.Stacks = append(b.Stacks, BlobStack{
Timing: timing,
OriginName: originName,
HostName: getHostName(),
})
return b
}
func (b BlobTrace) String() string {
var fullTrace string
for i, stack := range b.Stacks {
delta := time.Duration(0)
if i > 0 {
delta = stack.Timing - b.Stacks[i-1].Timing
}
fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
}
return fullTrace
}
func (b BlobTrace) Serialize() (string, error) {
t, err := json.Marshal(b)
if err != nil {
return "", errors.Err(err)
}
return string(t), nil
}
func Deserialize(serializedData string) (*BlobTrace, error) {
var trace BlobTrace
err := json.Unmarshal([]byte(serializedData), &trace)
if err != nil {
return nil, errors.Err(err)
}
return &trace, nil
}
+36
View File
@@ -0,0 +1,36 @@
package shared
import (
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/stretchr/testify/assert"
)
func TestBlobTrace_Serialize(t *testing.T) {
hostName = util.PtrToString("test_machine")
stack := NewBlobTrace(10*time.Second, "test")
stack.Stack(20*time.Second, "test2")
stack.Stack(30*time.Second, "test3")
serialized, err := stack.Serialize()
assert.NoError(t, err)
t.Log(serialized)
expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
assert.Equal(t, expected, serialized)
}
func TestBlobTrace_Deserialize(t *testing.T) {
hostName = util.PtrToString("test_machine")
serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
stack, err := Deserialize(serialized)
assert.NoError(t, err)
assert.Len(t, stack.Stacks, 3)
assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
assert.Equal(t, stack.Stacks[0].OriginName, "test")
assert.Equal(t, stack.Stacks[1].OriginName, "test2")
assert.Equal(t, stack.Stacks[2].OriginName, "test3")
}
+17
View File
@@ -0,0 +1,17 @@
// +build linux
package store
import (
"os"
"syscall"
"time"
)
func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
}
func atime(fi os.FileInfo) time.Time {
return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim)
}
+12
View File
@@ -0,0 +1,12 @@
// +build !linux
package store
import (
"os"
"time"
)
func atime(fi os.FileInfo) time.Time {
return fi.ModTime()
}
+107
View File
@@ -0,0 +1,107 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
type CachingStore struct {
origin, cache BlobStore
component string
}
// NewCachingStore makes a new caching disk store and returns a pointer to it.
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
return &CachingStore{
component: component,
origin: WithSingleFlight(component, origin),
cache: WithSingleFlight(component, cache),
}
}
const nameCaching = "caching"
// Name is the cache type name
func (c *CachingStore) Name() string { return nameCaching }
// Has checks the cache and then the origin for a hash. It returns true if either store has it.
func (c *CachingStore) Has(hash string) (bool, error) {
has, err := c.cache.Has(hash)
if has || err != nil {
return has, err
}
return c.origin.Has(hash)
}
// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
// from the origin, it is also stored in the cache.
func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cache.Get(hash)
if err == nil || !errors.Is(err, ErrBlobNotFound) {
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: c.cache.Name(),
metrics.LabelComponent: c.component,
metrics.LabelSource: "cache",
}).Set(rate)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
blob, trace, err = c.origin.Get(hash)
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
// do not do this async unless you're prepared to deal with mayhem
err = c.cache.Put(hash, blob)
if err != nil {
log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
}
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
// Put stores the blob in the origin and the cache
func (c *CachingStore) Put(hash string, blob stream.Blob) error {
err := c.origin.Put(hash, blob)
if err != nil {
return err
}
return c.cache.Put(hash, blob)
}
// PutSD stores the sd blob in the origin and the cache
func (c *CachingStore) PutSD(hash string, blob stream.Blob) error {
err := c.origin.PutSD(hash, blob)
if err != nil {
return err
}
return c.cache.PutSD(hash, blob)
}
// Delete deletes the blob from the origin and the cache
func (c *CachingStore) Delete(hash string) error {
err := c.origin.Delete(hash)
if err != nil {
return err
}
return c.cache.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *CachingStore) Shutdown() {
c.origin.Shutdown()
c.cache.Shutdown()
}
+178
View File
@@ -0,0 +1,178 @@
package store
import (
"bytes"
"sync"
"testing"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
func TestCachingStore_Put(t *testing.T) {
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
err := s.Put(hash, b)
if err != nil {
t.Fatal(err)
}
has, err := origin.Has(hash)
if err != nil {
t.Fatal(err)
}
if !has {
t.Errorf("failed to store blob in origin")
}
has, err = cache.Has(hash)
if err != nil {
t.Fatal(err)
}
if !has {
t.Errorf("failed to store blob in cache")
}
}
func TestCachingStore_CacheMiss(t *testing.T) {
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
err := origin.Put(hash, b)
if err != nil {
t.Fatal(err)
}
res, stack, err := s.Get(hash)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
}
time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
has, err := cache.Has(hash)
if err != nil {
t.Fatal(err)
}
if !has {
t.Errorf("Get() did not copy blob to cache")
}
t.Logf("stack: %s", stack.String())
res, stack, err = cache.Get(hash)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
}
t.Logf("stack: %s", stack.String())
}
func TestCachingStore_ThunderingHerd(t *testing.T) {
storeDelay := 100 * time.Millisecond
origin := NewSlowBlobStore(storeDelay)
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
err := origin.Put(hash, b)
if err != nil {
t.Fatal(err)
}
wg := &sync.WaitGroup{}
getNoErr := func() {
res, _, err := s.Get(hash)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
}
wg.Done()
}
start := time.Now()
wg.Add(4)
go func() {
go getNoErr()
time.Sleep(10 * time.Millisecond)
go getNoErr()
time.Sleep(10 * time.Millisecond)
go getNoErr()
time.Sleep(10 * time.Millisecond)
go getNoErr()
}()
wg.Wait()
duration := time.Since(start)
// only the first getNoErr() should hit the origin. the rest should wait for the first request to return
// once the first returns, the others should return immediately
// therefore, if the delay much longer than 100ms, it means subsequent requests also went to the origin
expectedMaxDelay := storeDelay + 5*time.Millisecond // a bit of extra time to let requests finish
if duration > expectedMaxDelay {
t.Errorf("Expected delay of at most %s, got %s", expectedMaxDelay, duration)
}
}
// SlowBlobStore adds a delay to each request
type SlowBlobStore struct {
mem *MemStore
delay time.Duration
}
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
return &SlowBlobStore{
mem: NewMemStore(),
delay: delay,
}
}
func (s *SlowBlobStore) Name() string {
return "slow"
}
func (s *SlowBlobStore) Has(hash string) (bool, error) {
time.Sleep(s.delay)
return s.mem.Has(hash)
}
func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
time.Sleep(s.delay)
return s.mem.Get(hash)
}
func (s *SlowBlobStore) Put(hash string, blob stream.Blob) error {
time.Sleep(s.delay)
return s.mem.Put(hash, blob)
}
func (s *SlowBlobStore) PutSD(hash string, blob stream.Blob) error {
time.Sleep(s.delay)
return s.mem.PutSD(hash, blob)
}
func (s *SlowBlobStore) Delete(hash string) error {
time.Sleep(s.delay)
return s.mem.Delete(hash)
}
func (s *SlowBlobStore) Shutdown() {
return
}
+110
View File
@@ -0,0 +1,110 @@
package store
import (
"io"
"io/ioutil"
"net/http"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// CloudFrontROStore reads from cloudfront. All writes panic.
type CloudFrontROStore struct {
endpoint string // cloudflare endpoint
}
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
return &CloudFrontROStore{endpoint: endpoint}
}
const nameCloudFrontRO = "cloudfront_ro"
// Name is the cache type name
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
// Has checks if the hash is in the store.
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
if err != nil {
return false, err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
case http.StatusOK:
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
start := time.Now()
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(start)
status, body, err := c.cfRequest(http.MethodGet, hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
case http.StatusOK:
b, err := ioutil.ReadAll(body)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
}
metrics.MtrInBytesS3.Add(float64(len(b)))
return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
default:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
}
}
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + hash
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
res, err := http.DefaultClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
}
return res.StatusCode, res.Body, nil
}
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontROStore) Shutdown() {
}
+62
View File
@@ -0,0 +1,62 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
type CloudFrontRWStore struct {
cf *ITTTStore
s3 *S3Store
}
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
// NOTE: It panics if either argument is nil.
func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
if cf == nil || s3 == nil {
panic("both stores must be set")
}
return &CloudFrontRWStore{cf: cf, s3: s3}
}
const nameCloudFrontRW = "cloudfront_rw"
// Name is the cache type name
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
// Has checks if the hash is in the store.
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
return c.cf.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cf.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
return c.s3.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
return c.s3.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *CloudFrontRWStore) Delete(hash string) error {
return c.s3.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontRWStore) Shutdown() {
c.s3.Shutdown()
c.cf.Shutdown()
}
+61 -29
View File
@@ -3,39 +3,66 @@ package store
import (
"encoding/json"
"sync"
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/lbry.go/extras/errors"
log "github.com/sirupsen/logrus"
)
// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
type DBBackedS3Store struct {
s3 *S3BlobStore
db *db.SQL
blockedMu sync.RWMutex
blocked map[string]bool
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
type DBBackedStore struct {
blobs BlobStore
db *db.SQL
blockedMu sync.RWMutex
blocked map[string]bool
deleteOnMiss bool
}
// NewDBBackedS3Store returns an initialized store pointer.
func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
return &DBBackedS3Store{s3: s3, db: db}
// NewDBBackedStore returns an initialized store pointer.
func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
}
const nameDBBacked = "db-backed"
// Name is the cache type name
func (d *DBBackedStore) Name() string { return nameDBBacked }
// Has returns true if the blob is in the store
func (d *DBBackedS3Store) Has(hash string) (bool, error) {
return d.db.HasBlob(hash)
func (d *DBBackedStore) Has(hash string) (bool, error) {
return d.db.HasBlob(hash, false)
}
// Get gets the blob
func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
return d.s3.Get(hash)
func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
has, err := d.db.HasBlob(hash, true)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
if !has {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
}
b, stack, err := d.blobs.Get(hash)
if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
e2 := d.Delete(hash)
if e2 != nil {
log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
}
}
return b, stack.Stack(time.Since(start), d.Name()), err
}
// Put stores the blob in the S3 store and stores the blob information in the DB.
func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
err := d.s3.Put(hash, blob)
func (d *DBBackedStore) Put(hash string, blob stream.Blob) error {
err := d.blobs.Put(hash, blob)
if err != nil {
return err
}
@@ -45,17 +72,17 @@ func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
// there is an error storing the blob information in the DB.
func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error {
var blobContents db.SdBlob
err := json.Unmarshal(blob, &blobContents)
if err != nil {
return err
return errors.Err(err)
}
if blobContents.StreamHash == "" {
return errors.Err("sd blob is missing stream hash")
}
err = d.s3.PutSD(hash, blob)
err = d.blobs.PutSD(hash, blob)
if err != nil {
return err
}
@@ -63,8 +90,8 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
return d.db.AddSDBlob(hash, len(blob), blobContents)
}
func (d *DBBackedS3Store) Delete(hash string) error {
err := d.s3.Delete(hash)
func (d *DBBackedStore) Delete(hash string) error {
err := d.blobs.Delete(hash)
if err != nil {
return err
}
@@ -73,7 +100,7 @@ func (d *DBBackedS3Store) Delete(hash string) error {
}
// Block deletes the blob and prevents it from being uploaded in the future
func (d *DBBackedS3Store) Block(hash string) error {
func (d *DBBackedStore) Block(hash string) error {
if blocked, err := d.isBlocked(hash); blocked || err != nil {
return err
}
@@ -85,13 +112,13 @@ func (d *DBBackedS3Store) Block(hash string) error {
return err
}
has, err := d.db.HasBlob(hash)
has, err := d.db.HasBlob(hash, false)
if err != nil {
return err
}
if has {
err = d.s3.Delete(hash)
err = d.blobs.Delete(hash)
if err != nil {
return err
}
@@ -106,7 +133,7 @@ func (d *DBBackedS3Store) Block(hash string) error {
}
// Wants returns false if the hash exists or is blocked, true otherwise
func (d *DBBackedS3Store) Wants(hash string) (bool, error) {
func (d *DBBackedStore) Wants(hash string) (bool, error) {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
return false, err
@@ -119,11 +146,11 @@ func (d *DBBackedS3Store) Wants(hash string) (bool, error) {
// MissingBlobsForKnownStream returns missing blobs for an existing stream
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
// like no blobs are missing
func (d *DBBackedS3Store) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
return d.db.MissingBlobsForKnownStream(sdHash)
}
func (d *DBBackedS3Store) markBlocked(hash string) error {
func (d *DBBackedStore) markBlocked(hash string) error {
err := d.initBlocked()
if err != nil {
return err
@@ -136,7 +163,7 @@ func (d *DBBackedS3Store) markBlocked(hash string) error {
return nil
}
func (d *DBBackedS3Store) isBlocked(hash string) (bool, error) {
func (d *DBBackedStore) isBlocked(hash string) (bool, error) {
err := d.initBlocked()
if err != nil {
return false, err
@@ -148,7 +175,7 @@ func (d *DBBackedS3Store) isBlocked(hash string) (bool, error) {
return d.blocked[hash], nil
}
func (d *DBBackedS3Store) initBlocked() error {
func (d *DBBackedStore) initBlocked() error {
// first check without blocking since this is the most likely scenario
if d.blocked != nil {
return nil
@@ -167,3 +194,8 @@ func (d *DBBackedS3Store) initBlocked() error {
return err
}
// Shutdown shuts down the store gracefully
func (d *DBBackedStore) Shutdown() {
d.blobs.Shutdown()
}
+234
View File
@@ -0,0 +1,234 @@
package store
import (
"bytes"
"crypto/sha512"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/brk0v/directio"
log "github.com/sirupsen/logrus"
"go.uber.org/atomic"
)
// DiskStore stores blobs on a local disk
type DiskStore struct {
// the location of blobs on disk
blobDir string
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
prefixLength int
// true if initOnce ran, false otherwise
initialized bool
concurrentChecks atomic.Int32
}
const maxConcurrentChecks = 30
// NewDiskStore returns an initialized file disk store pointer.
func NewDiskStore(dir string, prefixLength int) *DiskStore {
return &DiskStore{
blobDir: dir,
prefixLength: prefixLength,
}
}
const nameDisk = "disk"
// Name is the cache type name
func (d *DiskStore) Name() string { return nameDisk }
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (d *DiskStore) Has(hash string) (bool, error) {
err := d.initOnce()
if err != nil {
return false, err
}
_, err = os.Stat(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, errors.Err(err)
}
return true, nil
}
func (d *DiskStore) Size(hash string) (int64, error) {
stat, err := os.Stat(d.path(hash))
if err != nil {
return 0, err
}
return stat.Size(), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := d.initOnce()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
blob, err := ioutil.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
}
// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
// poor because there is a possible race condition between the check and the actual +1
if d.concurrentChecks.Load() < maxConcurrentChecks {
d.concurrentChecks.Add(1)
defer d.concurrentChecks.Sub(1)
hashBytes := sha512.Sum384(blob)
readHash := hex.EncodeToString(hashBytes[:])
if hash != readHash {
message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
log.Errorf("%s", message)
err := d.Delete(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
}
}
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
}
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
// Use directio writer
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer dio.Flush()
// Write the body to file
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
// PutSD stores the sd blob on the disk
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
return d.Put(hash, blob)
}
// Delete deletes the blob from the store
func (d *DiskStore) Delete(hash string) error {
err := d.initOnce()
if err != nil {
return err
}
has, err := d.Has(hash)
if err != nil {
return err
}
if !has {
return nil
}
err = os.Remove(d.path(hash))
return errors.Err(err)
}
// list returns the hashes of blobs that already exist in the blobDir
func (d *DiskStore) list() ([]string, error) {
err := d.initOnce()
if err != nil {
return nil, err
}
return speedwalk.AllFiles(d.blobDir, true)
}
func (d *DiskStore) dir(hash string) string {
if d.prefixLength <= 0 || len(hash) < d.prefixLength {
return d.blobDir
}
return path.Join(d.blobDir, hash[:d.prefixLength])
}
func (d *DiskStore) tmpDir(hash string) string {
return path.Join(d.blobDir, "tmp")
}
func (d *DiskStore) path(hash string) string {
return path.Join(d.dir(hash), hash)
}
func (d *DiskStore) tmpPath(hash string) string {
return path.Join(d.tmpDir(hash), hash)
}
func (d *DiskStore) ensureDirExists(dir string) error {
return errors.Err(os.MkdirAll(dir, 0755))
}
func (d *DiskStore) initOnce() error {
if d.initialized {
return nil
}
err := d.ensureDirExists(d.blobDir)
if err != nil {
return err
}
err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
if err != nil {
return err
}
d.initialized = true
return nil
}
// Shutdown shuts down the store gracefully
func (d *DiskStore) Shutdown() {
}
func (d *DiskStore) Sendfile(hash string, conn net.Conn) error {
f, err := os.Open(d.path(hash))
if err != nil {
return err
}
_, err = io.Copy(conn, f)
if err != nil {
return err
}
return nil
}
+45
View File
@@ -0,0 +1,45 @@
package store
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"testing"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, _, err := d.Get(hash)
assert.NoError(t, err)
assert.EqualValues(t, data, blob)
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
blob, _, err := d.Get("nonexistent")
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound))
}
-115
View File
@@ -1,115 +0,0 @@
package store
import (
"io/ioutil"
"os"
"path"
"github.com/lbryio/lbry.go/extras/errors"
)
// FileBlobStore is a local disk store.
type FileBlobStore struct {
dir string
initialized bool
}
// NewFileBlobStore returns an initialized file disk store pointer.
func NewFileBlobStore(dir string) *FileBlobStore {
return &FileBlobStore{dir: dir}
}
func (f *FileBlobStore) path(hash string) string {
return path.Join(f.dir, hash)
}
func (f *FileBlobStore) initOnce() error {
if f.initialized {
return nil
}
if stat, err := os.Stat(f.dir); err != nil {
if os.IsNotExist(err) {
err2 := os.Mkdir(f.dir, 0755)
if err2 != nil {
return err2
}
} else {
return err
}
} else if !stat.IsDir() {
return errors.Err("blob dir exists but is not a dir")
}
f.initialized = true
return nil
}
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (f *FileBlobStore) Has(hash string) (bool, error) {
err := f.initOnce()
if err != nil {
return false, err
}
_, err = os.Stat(f.path(hash))
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// Get returns the byte slice of the blob stored or will error if the blob doesn't exist.
func (f *FileBlobStore) Get(hash string) ([]byte, error) {
err := f.initOnce()
if err != nil {
return []byte{}, err
}
file, err := os.Open(f.path(hash))
if err != nil {
if os.IsNotExist(err) {
return []byte{}, errors.Err(ErrBlobNotFound)
}
return []byte{}, err
}
return ioutil.ReadAll(file)
}
// Put stores the blob on disk
func (f *FileBlobStore) Put(hash string, blob []byte) error {
err := f.initOnce()
if err != nil {
return err
}
return ioutil.WriteFile(f.path(hash), blob, 0644)
}
// PutSD stores the sd blob on the disk
func (f *FileBlobStore) PutSD(hash string, blob []byte) error {
return f.Put(hash, blob)
}
// Delete deletes the blob from the store
func (f *FileBlobStore) Delete(hash string) error {
err := f.initOnce()
if err != nil {
return err
}
has, err := f.Has(hash)
if err != nil {
return err
}
if !has {
return nil
}
return os.Remove(f.path(hash))
}
+9
View File
@@ -0,0 +1,9 @@
// +build darwin
package store
import (
"os"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
+10
View File
@@ -0,0 +1,10 @@
// +build linux
package store
import (
"os"
"syscall"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
+159
View File
@@ -0,0 +1,159 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/bluele/gcache"
"github.com/sirupsen/logrus"
)
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
type GcacheStore struct {
// underlying store
store BlobStore
// cache implementation
cache gcache.Cache
}
type EvictionStrategy int
const (
//LFU Discards the least frequently used items first.
LFU EvictionStrategy = iota
//ARC Constantly balances between LRU and LFU, to improve the combined result.
ARC
//LRU Discards the least recently used items first.
LRU
//SIMPLE has no clear priority for evict cache. It depends on key-value map order.
SIMPLE
)
// NewGcacheStore initialize a new LRUStore
func NewGcacheStore(component string, store BlobStore, maxSize int, strategy EvictionStrategy) *GcacheStore {
cacheBuilder := gcache.New(maxSize)
var cache gcache.Cache
evictFunc := func(key interface{}, value interface{}) {
logrus.Infof("evicting %s", key)
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
}
switch strategy {
case LFU:
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
case ARC:
cache = cacheBuilder.ARC().EvictedFunc(evictFunc).Build()
case LRU:
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
case SIMPLE:
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
}
l := &GcacheStore{
store: store,
cache: cache,
}
go func() {
if lstr, ok := store.(lister); ok {
err := l.loadExisting(lstr, int(maxSize))
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
}
}()
return l
}
const nameGcache = "gcache"
// Name is the cache type name
func (l *GcacheStore) Name() string { return nameGcache }
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *GcacheStore) Has(hash string) (bool, error) {
return l.cache.Has(hash), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
_, err := l.cache.Get(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.store.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.cache.Remove(hash)
}
return blob, stack.Stack(time.Since(start), l.Name()), err
}
// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.store.Put(hash, blob)
if err != nil {
return err
}
}
return nil
}
// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.store.PutSD(hash, blob)
if err != nil {
return err
}
}
return nil
}
// Delete deletes the blob from the store
func (l *GcacheStore) Delete(hash string) error {
err := l.store.Delete(hash)
if err != nil {
return err
}
// This must come after store.Delete()
// Remove triggers onEvict function, which also tries to delete blob from store
// We need to delete it manually first so any errors can be propagated up
l.cache.Remove(hash)
return nil
}
// loadExisting imports existing blobs from the underlying store into the LRU cache
func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("loading at most %d items", maxItems)
existing, err := store.list()
if err != nil {
return err
}
logrus.Infof("read %d files from underlying store", len(existing))
added := 0
for _, h := range existing {
_ = l.cache.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
break
}
}
return nil
}
// Shutdown shuts down the store gracefully
func (l *GcacheStore) Shutdown() {
}
+111
View File
@@ -0,0 +1,111 @@
package store
import (
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const cacheMaxSize = 3
func getTestGcacheStore() (*GcacheStore, *MemStore) {
m := NewMemStore()
return NewGcacheStore("test", m, cacheMaxSize, LFU), m
}
func TestGcacheStore_Eviction(t *testing.T) {
lfu, mem := getTestGcacheStore()
b := []byte("x")
for i := 0; i < 3; i++ {
err := lfu.Put(fmt.Sprintf("%d", i), b)
require.NoError(t, err)
for j := 0; j < 3-i; j++ {
_, _, err = lfu.Get(fmt.Sprintf("%d", i))
require.NoError(t, err)
}
}
for k, v := range map[string]bool{
"0": true,
"1": true,
"2": true,
} {
has, err := lfu.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
err := lfu.Put("3", b)
require.NoError(t, err)
for k, v := range map[string]bool{
"0": true,
"1": true,
"2": false,
"3": true,
} {
has, err := lfu.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
assert.Equal(t, cacheMaxSize, len(mem.Debug()))
err = lfu.Delete("0")
assert.NoError(t, err)
err = lfu.Delete("1")
assert.NoError(t, err)
err = lfu.Delete("3")
assert.NoError(t, err)
assert.Equal(t, 0, len(mem.Debug()))
}
func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
lfu, mem := getTestGcacheStore()
hash := "hash"
b := []byte("this is a blob of stuff")
err := lfu.Put(hash, b)
require.NoError(t, err)
err = mem.Delete(hash)
require.NoError(t, err)
// hash still exists in lru
assert.True(t, lfu.cache.Has(hash))
blob, _, err := lfu.Get(hash)
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
reflect.TypeOf(err).String(), err.Error())
// lru.Get() removes hash if underlying store doesn't have it
assert.False(t, lfu.cache.Has(hash))
}
func TestGcacheStore_loadExisting(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "hash"
b := []byte("this is a blob of stuff")
err = d.Put(hash, b)
require.NoError(t, err)
existing, err := d.list()
require.NoError(t, err)
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lfu := NewGcacheStore("test", d, 3, LFU) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lfu.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
}
+160
View File
@@ -0,0 +1,160 @@
package store
import (
"bytes"
"io"
"io/ioutil"
"net"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// HttpStore is a store that works on top of the HTTP protocol
type HttpStore struct {
upstream string
httpClient *http.Client
}
func NewHttpStore(upstream string) *HttpStore {
return &HttpStore{
upstream: "http://" + upstream,
httpClient: getClient(),
}
}
const nameHttp = "http"
func (n *HttpStore) Name() string { return nameHttp }
func (n *HttpStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer res.Body.Close()
tmp := getBuffer()
defer putBuffer(tmp)
serialized := res.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), n.Name())
if serialized != "" {
parsedTrace, err := shared.Deserialize(serialized)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
}
if res.StatusCode == http.StatusNotFound {
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
}
if res.StatusCode == http.StatusOK {
written, err := io.Copy(tmp, res.Body)
if err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
blob := make([]byte, written)
copy(blob, tmp.Bytes())
metrics.MtrInBytesHttp.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), n.Name()), nil
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *HttpStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return &http.Client{Transport: defaultTransport}
}
+73
View File
@@ -0,0 +1,73 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
type ITTTStore struct {
this, that BlobStore
}
// NewITTTStore returns a new instance of the IF THIS THAN THAT store
func NewITTTStore(this, that BlobStore) *ITTTStore {
return &ITTTStore{
this: this,
that: that,
}
}
const nameIttt = "ittt"
// Name is the cache type name
func (c *ITTTStore) Name() string { return nameIttt }
// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
func (c *ITTTStore) Has(hash string) (bool, error) {
has, err := c.this.Has(hash)
if err != nil || !has {
has, err = c.that.Has(hash)
}
return has, err
}
// Get tries to get the blob from this first, falling back to that.
func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.this.Get(hash)
if err == nil {
metrics.ThisHitCount.Inc()
return blob, trace.Stack(time.Since(start), c.Name()), err
}
blob, trace, err = c.that.Get(hash)
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
metrics.ThatHitCount.Inc()
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
// Put not implemented
func (c *ITTTStore) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD not implemented
func (c *ITTTStore) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete not implemented
func (c *ITTTStore) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *ITTTStore) Shutdown() {}
+47 -21
View File
@@ -1,54 +1,80 @@
package store
import "github.com/lbryio/lbry.go/extras/errors"
import (
"sync"
"time"
// MemoryBlobStore is an in memory only blob store with no persistence.
type MemoryBlobStore struct {
blobs map[string][]byte
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// MemStore is an in memory only blob store with no persistence.
type MemStore struct {
blobs map[string]stream.Blob
mu *sync.RWMutex
}
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemoryBlobStore) Has(hash string) (bool, error) {
if m.blobs == nil {
m.blobs = make(map[string][]byte)
func NewMemStore() *MemStore {
return &MemStore{
blobs: make(map[string]stream.Blob),
mu: &sync.RWMutex{},
}
}
const nameMem = "mem"
// Name is the cache type name
func (m *MemStore) Name() string { return nameMem }
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemStore) Has(hash string) (bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
_, ok := m.blobs[hash]
return ok, nil
}
// Get returns the blob byte slice if present and errors if the blob is not found.
func (m *MemoryBlobStore) Get(hash string) ([]byte, error) {
if m.blobs == nil {
m.blobs = make(map[string][]byte)
}
func (m *MemStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
m.mu.RLock()
defer m.mu.RUnlock()
blob, ok := m.blobs[hash]
if !ok {
return []byte{}, errors.Err(ErrBlobNotFound)
return nil, shared.NewBlobTrace(time.Since(start), m.Name()), errors.Err(ErrBlobNotFound)
}
return blob, nil
return blob, shared.NewBlobTrace(time.Since(start), m.Name()), nil
}
// Put stores the blob in memory
func (m *MemoryBlobStore) Put(hash string, blob []byte) error {
if m.blobs == nil {
m.blobs = make(map[string][]byte)
}
func (m *MemStore) Put(hash string, blob stream.Blob) error {
m.mu.Lock()
defer m.mu.Unlock()
m.blobs[hash] = blob
return nil
}
// PutSD stores the sd blob in memory
func (m *MemoryBlobStore) PutSD(hash string, blob []byte) error {
func (m *MemStore) PutSD(hash string, blob stream.Blob) error {
return m.Put(hash, blob)
}
// Delete deletes the blob from the store
func (m *MemoryBlobStore) Delete(hash string) error {
func (m *MemStore) Delete(hash string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.blobs, hash)
return nil
}
// Debug returns the blobs in memory. It's useful for testing and debugging.
func (m *MemoryBlobStore) Debug() map[string][]byte {
func (m *MemStore) Debug() map[string]stream.Blob {
m.mu.RLock()
defer m.mu.RUnlock()
return m.blobs
}
// Shutdown shuts down the store gracefully
func (m *MemStore) Shutdown() {}
+7 -7
View File
@@ -4,11 +4,11 @@ import (
"bytes"
"testing"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
func TestMemoryBlobStore_Put(t *testing.T) {
s := MemoryBlobStore{}
func TestMemStore_Put(t *testing.T) {
s := NewMemStore()
blob := []byte("abcdefg")
err := s.Put("abc", blob)
if err != nil {
@@ -16,8 +16,8 @@ func TestMemoryBlobStore_Put(t *testing.T) {
}
}
func TestMemoryBlobStore_Get(t *testing.T) {
s := MemoryBlobStore{}
func TestMemStore_Get(t *testing.T) {
s := NewMemStore()
hash := "abc"
blob := []byte("abcdefg")
err := s.Put(hash, blob)
@@ -25,7 +25,7 @@ func TestMemoryBlobStore_Get(t *testing.T) {
t.Error("error getting memory blob - ", err)
}
gotBlob, err := s.Get(hash)
gotBlob, _, err := s.Get(hash)
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
@@ -33,7 +33,7 @@ func TestMemoryBlobStore_Get(t *testing.T) {
t.Error("Got blob that is different from expected blob")
}
missingBlob, err := s.Get("nonexistent hash")
missingBlob, _, err := s.Get("nonexistent hash")
if err == nil {
t.Errorf("Expected ErrBlobNotFound, got nil")
}
+24
View File
@@ -0,0 +1,24 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// NoopStore is a store that does nothing
type NoopStore struct{}
const nameNoop = "noop"
func (n *NoopStore) Name() string { return nameNoop }
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
}
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) Delete(_ string) error { return nil }
func (n *NoopStore) Shutdown() { return }
+54 -36
View File
@@ -5,7 +5,11 @@ import (
"net/http"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -16,8 +20,8 @@ import (
log "github.com/sirupsen/logrus"
)
// S3BlobStore is an S3 store
type S3BlobStore struct {
// S3Store is an S3 store
type S3Store struct {
awsID string
awsSecret string
region string
@@ -26,9 +30,9 @@ type S3BlobStore struct {
session *session.Session
}
// NewS3BlobStore returns an initialized S3 store pointer.
func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore {
return &S3BlobStore{
// NewS3Store returns an initialized S3 store pointer.
func NewS3Store(awsID, awsSecret, region, bucket string) *S3Store {
return &S3Store{
awsID: awsID,
awsSecret: awsSecret,
region: region,
@@ -36,25 +40,13 @@ func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore {
}
}
func (s *S3BlobStore) initOnce() error {
if s.session != nil {
return nil
}
const nameS3 = "s3"
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
Region: aws.String(s.region),
})
if err != nil {
return err
}
s.session = sess
return nil
}
// Name is the cache type name
func (s *S3Store) Name() string { return nameS3 }
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
func (s *S3BlobStore) Has(hash string) (bool, error) {
func (s *S3Store) Has(hash string) (bool, error) {
err := s.initOnce()
if err != nil {
return false, err
@@ -75,17 +67,18 @@ func (s *S3BlobStore) Has(hash string) (bool, error) {
}
// Get returns the blob slice if present or errors on S3.
func (s *S3BlobStore) Get(hash string) ([]byte, error) {
func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
//Todo-Need to handle error for blob doesn't exist for consistency.
err := s.initOnce()
if err != nil {
return []byte{}, err
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
}
log.Debugf("Getting %s from S3", hash[:8])
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(time.Now())
}(start)
buf := &aws.WriteAtBuffer{}
_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
@@ -96,19 +89,19 @@ func (s *S3BlobStore) Get(hash string) ([]byte, error) {
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchBucket:
return []byte{}, errors.Err("bucket %s does not exist", s.bucket)
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case s3.ErrCodeNoSuchKey:
return []byte{}, errors.Err(ErrBlobNotFound)
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
}
}
return buf.Bytes(), err
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
}
return buf.Bytes(), nil
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
}
// Put stores the blob on S3 or errors if S3 connection errors.
func (s *S3BlobStore) Put(hash string, blob []byte) error {
func (s *S3Store) Put(hash string, blob stream.Blob) error {
err := s.initOnce()
if err != nil {
return err
@@ -120,22 +113,24 @@ func (s *S3BlobStore) Put(hash string, blob []byte) error {
}(time.Now())
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
StorageClass: aws.String(s3.StorageClassIntelligentTiering),
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
ACL: aws.String("public-read"),
//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
})
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
return err
}
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
func (s *S3BlobStore) PutSD(hash string, blob []byte) error {
func (s *S3Store) PutSD(hash string, blob stream.Blob) error {
//Todo - handle missing stream for consistency
return s.Put(hash, blob)
}
func (s *S3BlobStore) Delete(hash string) error {
func (s *S3Store) Delete(hash string) error {
err := s.initOnce()
if err != nil {
return err
@@ -150,3 +145,26 @@ func (s *S3BlobStore) Delete(hash string) error {
return err
}
func (s *S3Store) initOnce() error {
if s.session != nil {
return nil
}
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
Region: aws.String(s.region),
Endpoint: aws.String("https://s3.wasabisys.com"),
})
if err != nil {
return err
}
s.session = sess
return nil
}
// Shutdown shuts down the store gracefully
func (s *S3Store) Shutdown() {
return
}
+128
View File
@@ -0,0 +1,128 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"golang.org/x/sync/singleflight"
)
func WithSingleFlight(component string, origin BlobStore) BlobStore {
return &singleflightStore{
BlobStore: origin,
component: component,
sf: new(singleflight.Group),
}
}
type singleflightStore struct {
BlobStore
component string
sf *singleflight.Group
}
func (s *singleflightStore) Name() string {
return "sf_" + s.BlobStore.Name()
}
type getterResponse struct {
blob stream.Blob
stack shared.BlobTrace
}
// Get ensures that only one request per hash is sent to the origin at a time,
// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
gr, err, _ := s.sf.Do(hash, s.getter(hash))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
}
if gr == nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("getter response is nil")
}
rsp := gr.(getterResponse)
return rsp.blob, rsp.stack, nil
}
// getter returns a function that gets a blob from the origin
// only one getter per hash will be executing at a time
func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
return func() (interface{}, error) {
metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
start := time.Now()
blob, stack, err := s.BlobStore.Get(hash)
if err != nil {
return getterResponse{
blob: nil,
stack: stack.Stack(time.Since(start), s.Name()),
}, err
}
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: s.Name(),
metrics.LabelComponent: s.component,
metrics.LabelSource: "origin",
}).Set(rate)
return getterResponse{
blob: blob,
stack: stack.Stack(time.Since(start), s.Name()),
}, nil
}
}
// Put ensures that only one request per hash is sent to the origin at a time,
// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
func (s *singleflightStore) Put(hash string, blob stream.Blob) error {
metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
_, err, _ := s.sf.Do(hash, s.putter(hash, blob))
if err != nil {
return err
}
return nil
}
// putter returns a function that puts a blob from the origin
// only one putter per hash will be executing at a time
func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interface{}, error) {
return func() (interface{}, error) {
metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
start := time.Now()
err := s.BlobStore.Put(hash, blob)
if err != nil {
return nil, err
}
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: s.Name(),
metrics.LabelComponent: s.component,
metrics.LabelSource: "origin",
}).Set(rate)
return nil, nil
}
}
// Shutdown shuts down the store gracefully
func (s *singleflightStore) Shutdown() {
s.BlobStore.Shutdown()
return
}
+90
View File
@@ -0,0 +1,90 @@
package speedwalk
import (
"io/ioutil"
"path/filepath"
"runtime"
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/karrick/godirwalk"
"github.com/sirupsen/logrus"
)
// AllFiles recursively lists every file in every subdirectory of a given directory
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
func AllFiles(startDir string, basename bool) ([]string, error) {
items, err := ioutil.ReadDir(startDir)
if err != nil {
return nil, err
}
pathChan := make(chan string)
paths := make([]string, 0, 1000)
pathWG := &sync.WaitGroup{}
pathWG.Add(1)
metrics.RoutinesQueue.WithLabelValues("speedwalk", "worker").Inc()
go func() {
defer pathWG.Done()
for {
path, ok := <-pathChan
if !ok {
return
}
paths = append(paths, path)
}
}()
maxThreads := runtime.NumCPU() - 1
goroutineLimiter := make(chan struct{}, maxThreads)
for i := 0; i < maxThreads; i++ {
goroutineLimiter <- struct{}{}
}
walkerWG := &sync.WaitGroup{}
for _, item := range items {
if !item.IsDir() {
if basename {
pathChan <- item.Name()
} else {
pathChan <- filepath.Join(startDir, item.Name())
}
continue
}
<-goroutineLimiter
walkerWG.Add(1)
go func(dir string) {
defer func() {
walkerWG.Done()
goroutineLimiter <- struct{}{}
}()
err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{
Unsorted: true, // faster this way
Callback: func(osPathname string, de *godirwalk.Dirent) error {
if de.IsRegular() {
if basename {
pathChan <- de.Name()
} else {
pathChan <- osPathname
}
}
return nil
},
})
if err != nil {
logrus.Errorf(errors.FullTrace(err))
}
}(item.Name())
}
walkerWG.Wait()
close(pathChan)
pathWG.Wait()
return paths, nil
}
+27 -11
View File
@@ -1,27 +1,43 @@
package store
import "github.com/lbryio/lbry.go/extras/errors"
import (
"github.com/lbryio/reflector.go/shared"
// BlobStore is an interface with methods for consistently handling blob storage.
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// BlobStore is an interface for handling blob storage.
type BlobStore interface {
// Does blob exist in the store
// Name of blob store (useful for metrics)
Name() string
// Has Does blob exist in the store.
Has(hash string) (bool, error)
// Get the blob from the store
Get(hash string) ([]byte, error)
// Put the blob into the store
Put(hash string, blob []byte) error
// Put an SD blob into the store
PutSD(hash string, blob []byte) error
// Delete the blob from the store
// Get the blob from the store. Must return ErrBlobNotFound if blob is not in store.
Get(hash string) (stream.Blob, shared.BlobTrace, error)
// Put the blob into the store.
Put(hash string, blob stream.Blob) error
// PutSD an SD blob into the store.
PutSD(hash string, blob stream.Blob) error
// Delete the blob from the store.
Delete(hash string) error
// Shutdown the store gracefully
Shutdown()
}
// Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.
type Blocklister interface {
// Block deletes the blob and prevents it from being uploaded in the future
Block(hash string) error
// Wants returns false if the hash exists or is blocked, true otherwise
// Wants returns false if the hash exists in store or is blocked, true otherwise
Wants(hash string) (bool, error)
}
// lister is a store that can list cached blobs. This is helpful when an overlay
// cache needs to track blob existence.
type lister interface {
list() ([]string, error)
}
//ErrBlobNotFound is a standard error when a blob is not found in the store.
var ErrBlobNotFound = errors.Base("blob not found")
+106
View File
@@ -1,5 +1,24 @@
package wallet
import (
"encoding/base64"
"encoding/hex"
"github.com/lbryio/chainquery/lbrycrd"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/schema/stake"
types "github.com/lbryio/types/v2/go"
"github.com/btcsuite/btcutil"
"github.com/golang/protobuf/proto"
"github.com/spf13/cast"
)
// Raw makes a raw wallet server request
func (n *Node) Raw(method string, params []string, v interface{}) error {
return n.request(method, params, v)
}
// ServerVersion returns the server's version.
// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-version
func (n *Node) ServerVersion() (string, error) {
@@ -16,6 +35,38 @@ func (n *Node) ServerVersion() (string, error) {
return v, err
}
func (n *Node) Resolve(url string) (*types.Output, error) {
outputs := &types.Outputs{}
resp := &struct {
Result string `json:"result"`
}{}
err := n.request("blockchain.claimtrie.resolve", []string{url}, resp)
if err != nil {
return nil, err
}
b, err := base64.StdEncoding.DecodeString(resp.Result)
if err != nil {
return nil, errors.Err(err)
}
err = proto.Unmarshal(b, outputs)
if err != nil {
return nil, errors.Err(err)
}
if len(outputs.GetTxos()) != 1 {
return nil, errors.Err("expected 1 output, got " + cast.ToString(len(outputs.GetTxos())))
}
if e := outputs.GetTxos()[0].GetError(); e != nil {
return nil, errors.Err("%s: %s", e.GetCode(), e.GetText())
}
return outputs.GetTxos()[0], nil
}
type GetClaimsInTxResp struct {
Jsonrpc string `json:"jsonrpc"`
ID int `json:"id"`
@@ -41,3 +92,58 @@ func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
err := n.request("blockchain.claimtrie.getclaimsintx", []string{txid}, &resp)
return &resp, err
}
func (n *Node) GetTx(txid string) (string, error) {
resp := &struct {
Result string `json:"result"`
}{}
err := n.request("blockchain.transaction.get", []string{txid}, resp)
if err != nil {
return "", err
}
return resp.Result, nil
}
func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) {
hexTx, err := n.GetTx(txid)
if err != nil {
return nil, errors.Err(err)
}
rawTx, err := hex.DecodeString(hexTx)
if err != nil {
return nil, errors.Err(err)
}
tx, err := btcutil.NewTxFromBytes(rawTx)
if err != nil {
return nil, errors.Err(err)
}
if len(tx.MsgTx().TxOut) <= nout {
return nil, errors.Err("nout not found")
}
script := tx.MsgTx().TxOut[nout].PkScript
var value []byte
if lbrycrd.IsClaimNameScript(script) {
_, value, _, err = lbrycrd.ParseClaimNameScript(script)
} else if lbrycrd.IsClaimUpdateScript(script) {
_, _, value, _, err = lbrycrd.ParseClaimUpdateScript(script)
} else {
err = errors.Err("no claim found in output")
}
if err != nil {
return nil, errors.Err(err)
}
ch, err := stake.DecodeClaimBytes(value, "")
if err != nil {
return nil, errors.Err(err)
}
return ch.Claim, nil
}
+57 -18
View File
@@ -10,11 +10,11 @@ import (
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
"github.com/uber-go/atomic"
"go.uber.org/atomic"
)
const (
@@ -63,7 +63,7 @@ func NewNode() *Node {
// Connect creates a new connection to the specified address.
func (n *Node) Connect(addrs []string, config *tls.Config) error {
if n.transport != nil {
return ErrNodeConnected
return errors.Err(ErrNodeConnected)
}
// shuffle addresses for load balancing
@@ -76,15 +76,18 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error {
if err == nil {
break
}
if errors.Is(err, ErrTimeout) {
continue
}
if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" {
// net.errNoSuchHost is not exported, so we have to string-match
continue
}
return err
return errors.Err(err)
}
if n.transport == nil {
return ErrConnectFailed
return errors.Err(ErrConnectFailed)
}
log.Debugf("wallet connected to %s", n.transport.conn.RemoteAddr())
@@ -112,7 +115,13 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error {
}
func (n *Node) Shutdown() {
var addr net.Addr
if n.transport != nil {
addr = n.transport.conn.RemoteAddr()
}
log.Debugf("shutting down wallet %s", addr)
n.grp.StopAndWait()
log.Debugf("wallet stopped")
}
func (n *Node) handleErrors() {
@@ -121,7 +130,7 @@ func (n *Node) handleErrors() {
case <-n.grp.Ch():
return
case err := <-n.transport.Errors():
n.err(err)
n.err(errors.Err(err))
}
}
}
@@ -129,12 +138,18 @@ func (n *Node) handleErrors() {
// err handles errors produced by the foreign node.
func (n *Node) err(err error) {
// TODO: Better error handling.
log.Error(err)
log.Error(errors.FullTrace(err))
}
// listen processes messages from the server.
func (n *Node) listen() {
for {
select {
case <-n.grp.Ch():
return
default:
}
select {
case <-n.grp.Ch():
return
@@ -147,14 +162,36 @@ func (n *Node) listen() {
Message string `json:"message"`
} `json:"error"`
}{}
if err := json.Unmarshal(bytes, msg); err != nil {
n.err(err)
continue
msg2 := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"message"`
} `json:"error"`
}{}
r := response{}
err := json.Unmarshal(bytes, msg)
if err != nil {
// try msg2, a hack around the weird error-in-error response we sometimes get from wallet server
// maybe that happens because the wallet server passes a lbrycrd error through to us?
if err2 := json.Unmarshal(bytes, msg2); err2 == nil {
err = nil
msg.Id = msg2.Id
msg.Method = msg2.Method
msg.Error = msg2.Error.Message
}
}
r := response{}
if len(msg.Error.Message) > 0 {
r.err = errors.Base("%d: %s", msg.Error.Code, msg.Error.Message)
if err != nil {
r.err = errors.Err(err)
n.err(r.err)
} else if len(msg.Error.Message) > 0 {
r.err = errors.Err("%d: %s", msg.Error.Code, msg.Error.Message)
} else {
r.data = bytes
}
@@ -206,7 +243,7 @@ func (n *Node) request(method string, params []string, v interface{}) error {
bytes, err := json.Marshal(msg)
if err != nil {
return err
return errors.Err(err)
}
bytes = append(bytes, delimiter)
@@ -218,11 +255,13 @@ func (n *Node) request(method string, params []string, v interface{}) error {
err = n.transport.Send(bytes)
if err != nil {
return err
return errors.Err(err)
}
var r response
select {
case <-n.grp.Ch():
return nil
case r = <-c:
case <-time.After(n.timeout):
r = response{err: errors.Err(ErrTimeout)}
@@ -233,8 +272,8 @@ func (n *Node) request(method string, params []string, v interface{}) error {
n.handlersMu.Unlock()
if r.err != nil {
return r.err
return errors.Err(r.err)
}
return json.Unmarshal(r.data, v)
return errors.Err(json.Unmarshal(r.data, v))
}
+48 -12
View File
@@ -5,10 +5,14 @@ package wallet
import (
"bufio"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"time"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
)
@@ -49,15 +53,31 @@ func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) {
t.grp.Add(1)
go func() {
t.grp.Done()
defer t.grp.Done()
t.listen()
}()
err = t.test()
if err != nil {
t.grp.StopAndWait()
return nil, errors.Prefix(addr, err)
}
return t, nil
}
const delimiter = byte('\n')
func (t *TCPTransport) Send(body []byte) error {
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
_, err := t.conn.Write(body)
return err
}
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
func (t *TCPTransport) Errors() <-chan error { return t.errors }
func (t *TCPTransport) Shutdown() { t.grp.StopAndWait() }
func (t *TCPTransport) listen() {
reader := bufio.NewReader(t.conn)
for {
@@ -73,12 +93,6 @@ func (t *TCPTransport) listen() {
}
}
func (t *TCPTransport) Send(body []byte) error {
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
_, err := t.conn.Write(body)
return err
}
func (t *TCPTransport) error(err error) {
select {
case t.errors <- err:
@@ -86,11 +100,33 @@ func (t *TCPTransport) error(err error) {
}
}
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
func (t *TCPTransport) Errors() <-chan error { return t.errors }
func (t *TCPTransport) test() error {
err := t.Send([]byte(`{"id":1,"method":"server.version"}` + "\n"))
if err != nil {
return errors.Err(err)
}
func (t *TCPTransport) Shutdown() {
t.grp.StopAndWait()
var data []byte
select {
case data = <-t.Responses():
case <-time.Tick(1 * time.Second):
return errors.Err(ErrTimeout)
}
var response struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
err = json.Unmarshal(data, &response)
if err != nil {
return errors.Err(err)
}
if response.Error.Message != "" {
return fmt.Errorf(response.Error.Message)
}
return nil
}
func (t *TCPTransport) close() {