Compare commits

..
331 Commits
Author SHA1 Message Date
Alex Grintsvayg e554bbfe18 started work on local ytsync 2021-10-25 10:45:39 -04:00
Niko Storni 6944e17f43 fix livestreams failing for wrong reason 2021-08-24 12:03:03 -04:00
Niko Storni a224fe44c2 fix stream by magic 2021-08-24 11:54:38 -04:00
Niko Storni fa9dc35123 fix patched bug 2021-07-26 18:03:37 +02:00
Niko Storni 3a0882230a bypass throttling 2021-07-13 22:49:27 -04:00
Niko Storni fbd683e094 don't panic 2021-07-08 01:47:57 +02:00
Niko Storni 2f15c920d4 speed up playlist listing 2021-06-25 19:16:01 +02:00
Niko Storni 01f6448e72 revert aria2c usage 2021-06-25 19:09:00 +02:00
Niko Storni 8fb1e2ead0 fix empty thumbnails 2021-06-25 19:04:40 +02:00
Niko Storni 3b84db382c attempt using aria2c for some servers 2021-06-24 23:06:27 +02:00
Niko Storni 4fe6840a4e refactor code
fix bugs
add stub for speed checker
2021-06-18 04:47:08 +02:00
Niko Storni 69e6fb51d1 improve error handling 2021-06-18 03:09:19 +02:00
Niko Storni f17110ab7f fix nilptr 2021-06-18 01:23:25 +02:00
Niko Storni 768743a200 fix deadlock 2021-06-17 22:02:42 +02:00
Niko Storni 7c652b22a1 better output 2021-06-17 19:43:30 +02:00
Niko Storni a0fb4e579e fix progressbar
fix videos with leading dash
2021-06-17 19:13:44 +02:00
Niko Storni 519e1e4648 switch to yt-dlp
add progressbars
avoid unnecessary calls to youtube
update user agents
cookies fixes
bug fixes
introduction of new bugs
2021-06-17 17:51:21 +02:00
Niko Storni 087f20c133 fix broken video codec
exclude audio codecs that don't work on mobile
2021-05-25 21:08:12 +02:00
Niko Storni 21e6603a26 rollback is_spent changes
upgrade reflector/lbry.go
2021-04-13 00:55:50 +02:00
Niko Storni ca41f5052e upgrade go on travis 2021-04-12 23:41:47 +02:00
Niko Storni d739a98beb upgrade reflector to switch to wasabi 2021-04-12 23:31:37 +02:00
Niko Storni 7c7ceed333 further improve language detection
strip URLs from description to get better results
2021-03-25 19:07:26 +01:00
Niko Storni 198473b62b improve language detection 2021-03-25 18:47:34 +01:00
Niko Storni d53d0a1d52 fix issues with unicode claim names
fix issue with streams to abandon
2021-03-23 01:18:26 +01:00
Niko Storni 55577201a4 remove livelock 2021-03-17 16:40:37 +01:00
Niko Storni cacd21f840 fix bugs with claim names and unicode
update tests and improve logging
2021-03-15 22:50:23 +01:00
Niko Storni 9799b0a732 allow claim names with non latin chars
upgrade to latest lbrynet
improve e2e test
2021-03-04 03:04:52 +01:00
Niko Storni e985a60761 improve upload time detection 2021-03-01 23:34:19 +01:00
Niko Storni 4b4cee9fcc handle weird state 2021-02-25 05:05:28 +01:00
Niko Storni acca80dc70 fix unmarshal issue 2021-02-22 19:46:22 +01:00
Niko Storni 053c618458 fix another bug with youtube-dl 2021-02-22 17:29:52 +01:00
Niko Storni 2cf3526c74 switch back to youtube-dl for downloads only 2021-02-18 23:28:55 +01:00
Niko Storni d7e194cb5c update chrome UA
change throttle timeout to 48 hours
2021-02-11 23:44:48 +01:00
Niko Storni 9a8b1922fe fix channel list 2021-02-11 21:43:58 +01:00
Niko Storni 12d627bd35 fix stupid error 2021-01-12 16:49:54 +01:00
Niko Storni 9397a40c9f disable unsent tips check as we don't really use it now
wait a little bit longer on new blocks
2020-12-16 16:59:07 +01:00
Niko Storni 6a4093f3f8 fix youtube channel info parsing 2020-11-27 13:12:01 -05:00
Niko Storni bac2969295 prevent panic failure 2020-11-19 15:58:31 +01:00
Niko Storni 5dfd8dee1b change default videos to sync
add length check for short videos
potentially fix blockchain.db issue
2020-11-19 03:11:23 +01:00
Niko Storni 06c43c3f71 use youtube-dlc instead 2020-11-10 18:18:07 +01:00
Niko Storni 881d86799b improve logging 2020-11-04 18:13:30 +01:00
Niko Storni f7e4366a67 fix logging 2020-11-04 17:22:27 +01:00
Niko Storni 5b5f7e1c68 use video hard failures 2020-11-03 22:03:38 +01:00
Niko Storni fecf67118c remove unused flag
always go through syncing first
don't sync videos shorter than 7 seconds
refactor code in video error handling
add interface to handle hard video failures (incomplete)
2020-11-03 21:41:39 +01:00
Niko Storni beade71aa6 fix bugs 2020-11-03 02:14:01 +01:00
Niko Storni ca8ff505d4 add status to list
resume on disk usage overflow
2020-10-27 20:57:32 +01:00
Niko Storni f5f12e1560 add support for multiple queues
add support for blockchain.db pruning via new status
2020-10-27 19:50:10 +01:00
Niko Storni c4693c221f update lbrynet support
avoid failures when interrupted by user
2020-10-22 23:14:21 +02:00
Niko Storni 692c796770 handle non existing iso639-1 better 2020-10-20 17:43:25 +02:00
Niko Storni de798ade36 fix canceled by stopper error 2020-10-07 03:40:19 +02:00
Niko Storni a93c2c4538 add support for own S3 thumbnails storage 2020-10-07 03:10:54 +02:00
Niko Storni 53e0b7c47b fix bug that messes with counts 2020-09-25 21:44:14 +02:00
Niko Storni 070dda8739 add language detection for videos 2020-09-21 19:12:23 +02:00
Niko Storni f773569920 Merge branch 'fuck-yt' 2020-09-01 20:50:31 +02:00
Niko Storni ad6fa4d725 update dependencies 2020-09-01 20:27:39 +02:00
Niko Storni d4ca71a89d fix channel description 2020-08-31 21:27:39 +02:00
Niko Storni c53cf4c1b3 prevent channels from failing when manually killed 2020-08-26 19:05:16 +02:00
Niko Storni ad5a30da9e work around transfer failures 2020-08-25 18:44:44 +02:00
Niko Storni ecda80b02d bypass socialblade block
stop using socialblade for now
2020-08-21 00:15:14 +02:00
Niko Storni 24cf937e14 retry on support list failures 2020-08-18 01:09:53 +02:00
Niko Storni 0b002c8228 lots of bug fixes and improvements 2020-08-18 00:03:38 +02:00
Niko Storni a56166ee51 refactor youtube-dl execution process 2020-08-12 19:44:57 +02:00
Niko Storni ddca850c17 nevermind... re-enable ipv6 (i should make it a flag) 2020-08-12 04:17:30 +02:00
Niko Storni 31ce612e2e temporarily disable ipv6 2020-08-12 03:38:52 +02:00
Niko Storni eb8900c66a don't waste loops
fix a deadlock?
2020-08-12 03:37:18 +02:00
Niko Storni 67da4142d5 fix old transferred channels 2020-08-10 18:33:05 +02:00
Niko Storni fb0e567caf be more specific on video failures 2020-08-08 03:29:57 +02:00
Niko Storni 7e83b17b69 add fallback for socialblade API failures 2020-08-08 02:31:26 +02:00
Niko Storni 7c02c5b92d refactor structures
fix various bugs
2020-08-08 01:12:55 +02:00
Niko Storni 2a33f44317 fix e2e tests 2020-08-07 15:24:13 +02:00
Niko Storni b7037900f8 fix thumbnail download 2020-08-06 23:53:03 +02:00
Niko Storni 5d230a6b54 potentially fix looping
add limits by db
2020-08-06 20:32:49 +02:00
Niko Storni f0280b51b4 fix time parsing for real 2020-08-06 02:12:05 +02:00
Niko Storni 775e4881cb invert logic so that we don't flood slack 2020-08-06 00:30:48 +02:00
Niko Storni ec9f46f552 fix release date bug 2020-08-06 00:25:54 +02:00
Alex Grintsvayg 6c6e93cefc go mod tidy 2020-08-05 15:48:13 -04:00
Niko Storni eab9bcf1ff don't wait for confirmations unnecessarily
fix sqlite crap
2020-08-05 15:47:53 -04:00
Niko Storni 751bc84ce5 grab channel info without APIs
fix new channel syncs
fix e2e tests
2020-08-05 15:47:53 -04:00
Mark Beamer Jr 41fd9f6844 fix string parameter for error message 2020-08-05 15:47:53 -04:00
Mark Beamer Jr 7f1906d58b change FPS to float from int 2020-08-05 15:47:53 -04:00
Mark Beamer Jr 5be3551abe Mark video failed if we cannot get video info instead of failing the sync
Turn on error log after fixing lockup
Try different user agents if calls fail
Prevent double processing a channel in a row, just fail the sync instead
2020-08-05 15:47:53 -04:00
Mark Beamer Jr fc18151d77 Increase max db limit
Increase wallet startup time allotment for big channels
Set FPS to float instead of int
2020-08-05 15:47:53 -04:00
Mark Beamer Jr 13543b20e9 Don't use api unless the video is from the past 5 days, otherwise just use the upload date from youtube donwloader
change interface to int for now known field.
2020-08-05 15:47:53 -04:00
Mark Beamer Jr 4eba7200d8 user agent breaks it 2020-08-05 15:47:53 -04:00
Mark Beamer Jr cd11e82676 Dont always fall through on every error.
Add user agent to calls
2020-08-05 15:47:53 -04:00
Mark Beamer Jr c7c220ecd3 Dont always fall through on every error. 2020-08-05 15:47:53 -04:00
Mark Beamer Jr 4d56339756 fixed appending args 2020-08-05 15:47:53 -04:00
Mark Beamer Jr 140353097e added logging 2020-08-05 15:47:53 -04:00
Mark Beamer Jr a1caea4a28 added logging 2020-08-05 15:47:53 -04:00
Mark Beamer Jr b1e10e7b09 Add retry on IP failures 2020-08-05 15:47:53 -04:00
Mark Beamer Jr ad27425471 Add ip switching for being blocked 2020-08-05 15:47:53 -04:00
Mark Beamer Jr 81e9378b9d Add new fall through errors for release time 2020-08-05 15:47:53 -04:00
Mark Beamer Jr 64040ea67a Add calls to internal-apis and add get released date if available. 2020-08-05 15:47:47 -04:00
Mark Beamer Jr d99e200178 Add check for already published videos using new video-state api. 2020-07-29 00:12:23 -04:00
Alex Grintsvayg 357aebbcce less loud 2020-07-28 22:40:56 -04:00
Alex Grintsvayg 0eef62b5fd i made a huge mess. sorry future me, when you're digging through this 2020-07-28 21:34:08 -04:00
Alex Grintsvayg a3dd3dc626 WIP: trying to get the accurate api to work 2020-07-28 12:47:28 -04:00
Alex Grintsvayg f942bf8025 get upload time from https://www.invid-project.eu/tools-and-services/invid-verification-plugin/ 2020-07-28 11:20:22 -04:00
Alex Grintsvayg a05864404d use local timezone for youtube-dl, even though its not certain 2020-07-27 20:59:47 -04:00
Alex Grintsvayg 70ad891dfa use durations 2020-07-27 19:35:07 -04:00
Mark Beamer Jr 8e61cde0a0 turn of std error 2020-07-27 17:31:46 -04:00
Alex Grintsvayg 8600077caa no panics. thanks Beam-master 2020-07-27 17:27:50 -04:00
Mark Beamer Jr e7fdd21bac commit some more stuff 2020-07-27 17:19:15 -04:00
Mark Beamer Jr 0bb6b6d833 commit my shit 2020-07-27 17:14:18 -04:00
Alex Grintsvayg b59ef28267 punting on ChannelInfo for now 2020-07-27 16:02:52 -04:00
Mark Beamer Jr e926a2c1f6 Fix quickSync && maxVideos 2020-07-27 15:59:14 -04:00
Alex Grintsvayg 1369ed0b48 replace yt channel video count with a scrape of socialblade.com 2020-07-27 15:51:06 -04:00
Mark Beamer Jr f39fc11697 Add downloader to create api
adjust GetVideosToSync
2020-07-27 15:42:45 -04:00
Alex Grintsvayg bffc0823be more 2020-07-27 14:52:21 -04:00
Alex Grintsvayg edbb22fcf0 rename 2020-07-27 14:49:37 -04:00
Alex Grintsvayg 843303301a move all youtube api calls into a single place 2020-07-27 14:48:05 -04:00
Niko Storni be7fd7ddd8 Merge branch 'latest-lbry.go' 2020-07-21 16:59:26 -04:00
Niko Storni bfcfe5f36b fix e2e 2020-07-21 16:59:09 -04:00
Alex Grintsvayg 4c4da93ef3 bump lbry.go dependency to be compatible with internal-apis dependency bump 2020-07-21 13:55:34 -04:00
Niko Storni c2b5878daa don't delete lbry first videos from the db 2020-07-21 13:54:28 -04:00
Niko Storni 749d5095c9 fix semver 2020-06-11 18:45:56 +02:00
Niko ba77b61ae0 Merge pull request #56 from lbryio/fix-travis
fix travis try 2
2020-06-09 21:48:40 -04:00
Niko Storni 07915ce697 fix travis try 2 2020-06-10 03:44:34 +02:00
Niko Storni 4b2221ce5e fix travis (try 1) 2020-06-10 03:38:42 +02:00
Niko Storni 1d5b69bfe6 add automatic sleep until quota reset
re-enable ipv6
2020-06-10 03:32:45 +02:00
Niko Storni 4d915587e0 temporarily disable ipv6 2020-05-30 02:18:42 +02:00
Niko Storni 83f2c28c20 fix nil ptr 2020-05-22 12:47:18 -04:00
Niko Storni 47e467057d change timeouts to avoid long wait times 2020-05-20 04:28:29 +02:00
Niko Storni 1ae32d638b fix prom allowed chars 2020-05-19 23:24:16 +02:00
Niko Storni 7907ee3579 add prometheus metrics 2020-05-19 23:13:01 +02:00
Niko Storni 414ed1c130 or to and 2020-05-14 17:28:15 +02:00
Niko Storni 3bb17b09d3 fix never ending condition 2020-05-14 06:47:19 +02:00
Niko Storni 046f46267e update lbry.go dependency 2020-05-14 05:49:49 +02:00
Niko Storni adc3ffe194 update lbrynet support 2020-05-14 04:04:52 +02:00
Niko Storni 76301d40ba only wait a maximum of 2 blocks before transfering
fix debug string
2020-05-08 16:33:29 +02:00
Niko Storni 963c450730 allow all IPv6 to be used 2020-05-05 20:13:40 +02:00
Niko Storni 6e7bb994d0 don't keep reusing the same IP for updates 2020-05-05 05:58:34 +02:00
Niko Storni 95890cbabf prevent live streams from syncing
reduce timeout to 20 minutes
2020-04-29 16:15:22 +02:00
Niko Storni 69c2a91ec7 fix bug in video length 2020-04-23 16:37:35 +02:00
Niko Storni fbe09a692c fix 403 error 2020-04-21 20:56:14 +02:00
Niko Storni 0ccc6e7979 add temporary logging 2020-04-21 16:26:34 +02:00
Niko Storni 3eb74cc8b5 log video length failures 2020-04-19 02:38:31 +02:00
Niko Storni 9bfb1656fe fix length checks on single syncs 2020-04-10 17:46:07 +02:00
Niko Storni 5d40d24804 fix max length issue for bigger youtubers
fix issue with no-playlist-found channels
reduce the max db size limit
2020-04-08 23:14:10 +02:00
Niko Storni 79d3aadda8 add utxo_release as hotfix for stuck balances 2020-04-01 04:49:24 +02:00
Niko Storni eba59a0806 change utxo_list signature 2020-04-01 04:44:05 +02:00
Niko Storni 3fd4909cd1 replace support abandon with txo_spend
upgrade lbry.go library
upgrade lbrynet support
possibly introduce new bugs ;)
2020-03-31 04:30:02 +02:00
Niko Storni b59372c1d8 don't prevent channels with 0 youtube videos from transferring 2020-03-31 02:45:53 +02:00
Niko Storni c9e713e9df do not attempt upgrading channels no longer in our control 2020-03-27 18:16:35 +01:00
Niko Storni bc72d2f1c4 do not fail if database is missing 2020-03-25 18:28:42 +01:00
Niko Storni df539e5d01 fix ledger mess 2020-03-25 18:12:50 +01:00
Niko Storni 1b89104101 clean up blockchain.db if it grows too big 2020-03-25 17:53:47 +01:00
Niko Storni 3dc30c3b98 speed up duration checks 2020-03-12 19:22:23 +01:00
Niko Storni 259df8d257 restrict length amount on small youtubers 2020-03-12 04:08:50 +01:00
Niko Storni 5460224cd6 upgrade SDK
fix UTXO creation time waste
2020-02-24 15:52:32 +01:00
Niko Storni 29479a73a7 update lbry.go/lbrycrd dependencies 2020-02-11 20:24:05 +01:00
Niko Storni 625f339709 Merge branch 'latest_lbrycrd_support' 2020-02-11 20:22:59 +01:00
Mark Beamer Jr d6fcfd888a Upgrade to latest lbrycrd support for internal-apis 2020-02-11 20:22:50 +01:00
Thomas Zarebczan 5f7d9d4940 Merge pull request #51 from ykris45/patch-1
Update LICENSE
2020-02-03 17:03:52 -05:00
YULIUS KURNIAWAN KRISTIANTO 826fcb78ea Update LICENSE 2020-02-03 04:08:28 +07:00
Niko Storni 45e87c3007 don't ever re-create a channel if one was previously created 2020-01-14 04:34:33 +01:00
Niko Storni cdee0d0de7 fix typo 2020-01-14 04:14:14 +01:00
Niko Storni e30f21dc85 fail sync when channel is abandoned 2020-01-14 04:10:20 +01:00
Niko Storni df88c7e952 update docker compose 2020-01-14 03:40:30 +01:00
Niko Storni 62a1a0eed5 update goreleaser 2020-01-11 22:11:44 -05:00
Niko Storni 93f9185c71 fix bug when publishing 2020-01-11 22:01:40 -05:00
Niko Storni ef62242f42 add cookies to youtube-dl
minor fixes
2020-01-03 15:59:05 +01:00
Niko Storni 4a356e86da add quicksync for better quota usage 2019-12-27 18:12:41 +01:00
Niko Storni ac95ca57af halt when quota is exceeded 2019-12-27 04:15:48 +01:00
Niko Storni c28d5a716b send warning to slack when video processing panics 2019-12-27 01:39:27 +01:00
Niko Storni 59471f9c26 reduce sync delay
fix stopper bug
2019-12-27 01:27:29 +01:00
Niko Storni 8028c6621c add temporary workaround to release all IPs
speed up one failure
2019-12-26 17:52:44 +01:00
Niko Storni 8abf6ad255 add debug statement 2019-12-24 23:27:16 +01:00
Niko Storni 1f9a645c57 support create fixes
various bug fixes
2019-12-24 05:00:16 +01:00
Niko Storni a5657aaf46 refactor download
get rid of goto
2019-12-20 18:49:33 +01:00
Niko Storni b6861dae9b don't fail on abandoned videos 2019-12-19 03:13:22 +01:00
Niko Storni cff147bb95 work around missing inputs error 2019-12-18 20:24:50 +01:00
Niko Storni 54d04dcf2c add IP debugging
improve process halting
rename improper language code for hebrew
2019-12-18 18:22:15 +01:00
Niko Storni 65c4f99c1c make sure abandons spend from default account only 2019-12-16 04:53:15 +01:00
Niko Storni 68c0fd9ed7 add new throttling detection 2019-12-16 00:29:56 +01:00
Niko Storni 2bde06e4b9 fixes to throttling 2019-12-14 14:58:04 +01:00
Niko Storni 5df6db6e96 fix bug with non-related publishes 2019-12-14 06:43:01 +01:00
Niko Storni 62dfdc1adb fix possible resource leak 2019-12-14 05:49:25 +01:00
Niko Storni fe55304184 fix anti-throttling
fix duplicates with multi-channels
2019-12-13 18:20:41 +01:00
Mark Beamer Jr bb76500f04 Update e2e test for publish table migration in internal-apis 2019-12-12 22:28:19 -05:00
Niko Storni 4e2efec66e disable logging for reflection 2019-12-11 13:13:30 +01:00
Mark 8d650e032c Merge pull request #49 from lbryio/merge_e2e
Merge chainquery and lbry databases
2019-12-10 20:01:26 -05:00
Mark Beamer Jr f5d97957e7 Fixed the e2e test. It wasn't checking the right claims and videos. Looks like a copy paste error. 2019-12-10 18:43:44 -05:00
Niko Storni 36f539ef5d change ip throttling management 2019-12-10 23:02:56 +01:00
Mark Beamer Jr 77461f71af Merge chainquery and lbry databases to account for claim table being required by internal-apis due to replication changes. 2019-12-10 15:45:29 -05:00
Niko Storni caca13de61 increase anti-throttle timeout 2019-12-09 18:09:31 +01:00
Niko Storni 7bdc117ba0 upgrade ytsync to latest SDK
upgrade e2e tests
2019-12-08 11:49:27 -05:00
Niko Storni fc575dac80 improve logging
fix ipv6 switching
2019-12-08 10:31:15 -05:00
Niko Storni d520ff2c68 change cache control for thumbnails 2019-11-11 15:11:34 -05:00
Niko Storni 25e49f8ff2 fix live lock 2019-10-25 18:35:09 +02:00
Niko Storni 77d0efcf3c refactor tags_mapping name
remove failure state for transfers
2019-10-21 15:44:24 +02:00
Niko Storni 291a105269 improve getDefaultAccount
use default account on more calls
improve API logging
fix memory leak
2019-10-16 19:38:45 +02:00
Thomas Zarebczan 682117a030 Merge pull request #45 from StrikerRUS/license
added license file
2019-10-15 11:52:59 -04:00
Niko Storni 6d2e4aa6e6 list all claims at once (faster) 2019-10-15 00:29:26 +02:00
Niko Storni 77749f05ba APIs hiccups will no longer make ytsync crash 2019-10-14 17:51:26 +02:00
Niko Storni 10a855a8ba fix nil ptr dereference
fix bug in transfers
2019-10-12 09:49:49 +02:00
Niko Storni a9e76149e8 parallelize support abandons
update lbrynet support in e2e test
stop using reflector on regtest!
change e2e channel target for more videos
possibly fix a deadlock
2019-10-11 19:42:03 +02:00
Niko Storni dd4aebdba4 make video transferring async
fix bug in videosLimit
change test subject to have more videos
2019-10-11 01:00:04 +02:00
Niko Storni b409d36de0 remove unnecessary output 2019-10-10 18:17:13 +02:00
Niko Storni 30af4a0136 update lbry.go to v2 library
update modules
add slightly more LBCs for the channel claim to ensure fees are covered
2019-10-10 16:51:40 +02:00
StrikerRUS a3b660fab5 aadded license file 2019-10-09 00:11:55 +03:00
Niko Storni 65daeec216 resolve issue with channels attempting re-transfer 2019-10-08 14:42:57 +02:00
Niko Storni ad1704fb39 remove obsolete check 2019-10-08 02:13:17 +02:00
Niko Storni 44e6cb5ddc fix bug where newly published and transferred videos caused an error
rename variable for better clarity
2019-10-08 01:59:18 +02:00
Niko Storni 5171acc007 refactor integrity check
add checks for improper transfer state
add checks for unsent supports
only use default account for operations that only work on it
add more logging
2019-10-08 01:38:39 +02:00
Niko Storni 946314da94 update lbry.go to fix claim_search 2019-10-03 22:51:21 +02:00
Niko Storni 5961327adf reduce bid even further to avoid lockups 2019-10-03 21:06:47 +02:00
Niko Storni d2bbb36312 improve detection for own claims vs their claims 2019-10-01 16:55:43 +02:00
Niko Storni c7272e988a reduce amount to avoid dust 2019-09-27 20:17:49 +02:00
Niko Storni 5b9846bcca fix pointer issue 2019-09-27 19:29:42 +02:00
Niko Storni 8d849b8f06 do not retry unavailable videos 2019-09-27 16:12:21 +02:00
Niko Storni c41c368f34 fix transfer bugs 2019-09-27 16:10:57 +02:00
Niko Storni 7ce4c4a2ce fix lbrynet shutdown check in e2e 2019-09-25 05:07:19 +02:00
Niko Storni 0d473cc4f6 add support for transferring videos after transferring everything 2019-09-25 04:38:49 +02:00
Niko Storni 2f701a887a fix double counting
fix transferred check
2019-09-25 04:15:16 +02:00
Niko Storni 8d56ef852f fix e2e
fix database transferred flag
2019-09-25 03:30:07 +02:00
Niko Storni 849ff11bfd import public keys if available 2019-09-24 20:42:17 +02:00
Niko Storni 45982f30c5 fix e2e test 2019-09-24 19:09:39 +02:00
Niko Storni 07413b499a update SDK support
fix bug where videos would publish to the wrong address
2019-09-24 18:51:06 +02:00
Niko Storni 92d8ef75f8 update e2e test 2019-09-19 20:03:55 +02:00
Niko Storni 1b08bb0e61 fix bug with non latin titles for claim names
hard exit when a duplicate name is attempted
2019-09-10 11:43:20 +02:00
Niko Storni bdd55c9965 fix balance refill issue
add additional debugging
2019-09-04 19:24:11 +02:00
Niko Storni 14668c339e refactor sync flags
add disable-transfer flag
2019-08-30 21:31:58 +02:00
Niko Storni 7d38aa7b29 add republishing on non-available videos
enhanced logging
2019-08-30 19:35:04 +02:00
Niko Storni 3c3ea2138e adjust support management
adjust e2e script
2019-08-28 21:08:25 +02:00
Niko Storni 7b23235f83 Merge branch 'transfers' 2019-08-28 15:26:38 +02:00
Niko Storni 79ae630bc4 update reflector lib 2019-08-28 15:25:53 +02:00
Niko Storni 0d8db26e96 use the right errors lib
update dependencies
2019-08-28 15:25:53 +02:00
Mark Beamer Jr bacb91e82a Add supporty to create supports
Update lbry.go dep to get lbrycrd commands
2019-08-28 15:25:53 +02:00
Niko Storni 577411cf50 fix e2e
add support for supports (heh)
2019-08-28 15:25:53 +02:00
Mark Beamer Jr e98061f2eb Add chainquery to youtube sync repo to be used for e2e testing of support transfer
Remove timeout for wait-for-it calls
2019-08-28 15:25:53 +02:00
Mark Beamer Jr 4548c41082 Add chainquery to youtube sync repo 2019-08-28 15:25:53 +02:00
Niko Storni f7c80c2e5d refactor transfer process
start working on supports
2019-08-28 15:25:53 +02:00
Mark Beamer Jr 2d6e53be32 Added creation of supports for the e2e test 2019-08-28 15:25:53 +02:00
Mark Beamer Jr 419434db28 Add success message when the e2e test passes. 2019-08-28 15:25:53 +02:00
Mark Beamer Jr cd6ddaaa85 Update lbrynet to match wallet server version. 2019-08-28 15:25:53 +02:00
Niko Storni 4d16a1d6e8 fix e2e tests
update wallet server
cleanup go.sum
Fix broken balance logging
Prevent failures once channel is transferred
Wait for all UTXOs to confirm before transferring
Fix various bugs
2019-08-28 15:25:53 +02:00
Mark Beamer Jr 1fb1077a94 Update curl call to show response
Update channel status call to pass the correct name of the parameter
2019-08-28 15:25:53 +02:00
Mark Beamer Jr 2c9c999c71 Add marking video status when transfer of claim is complete.
Add error handling when transfer of claim fails.
2019-08-28 15:25:53 +02:00
Mark Beamer Jr c754562f96 Update to leverage lbrycrd v0.17.2.1, lbrynet v0.39.1, walletserver v0.39.1 2019-08-28 15:25:53 +02:00
Niko Storni 7a7de03c0f adjust account balance
minor fix
2019-08-28 15:25:53 +02:00
Niko Storni 1b9ed266e0 move channel status call to proper region
fix shadowed errors
refactor code to be more readable and consistent
get video transfer state
2019-08-28 15:25:53 +02:00
Mark Beamer Jr eb2f6273e4 Add transfer of channel after all video's have been transferred
Add channel status call to notify external db that the channel has been transferred.
2019-08-28 15:25:53 +02:00
Mark Beamer Jr 45cea808ed Add marking video status when transfer of claim is complete.
Add error handling when transfer of claim fails.
2019-08-28 15:25:53 +02:00
Mark Beamer Jr 9a5b6b4e56 Update to leverage lbrycrd v0.17.2.1, lbrynet v0.39.1, walletserver v0.39.1 2019-08-28 15:25:53 +02:00
Mark Beamer Jr d63aba568d Add transfer logic as post sync process step.
Add authtoken for youtuber in test data setup to call transfer api
Update e2e test to include scenario where we transfer channel.
2019-08-28 15:25:53 +02:00
Niko Storni 4cc0b71279 process failures sequentially 2019-08-20 11:33:06 +02:00
Niko Storni b8d66cb8c0 use latest lbry.go/lbrycrd release 2019-08-20 01:37:13 +02:00
Niko Storni eff76d22eb update lbry.go version 2019-08-16 17:45:19 +02:00
Niko Storni 8ebdf33213 update lbry.go 2019-08-16 13:49:31 +02:00
Niko Storni 81616c1d96 refactor channel selector
remove unused code
2019-08-15 19:31:43 +02:00
Niko Storni b477d18a16 fix claim_search 2019-08-14 17:44:39 +02:00
Niko Storni 5cb7f72ef7 remove useless logging 2019-08-14 05:18:57 +02:00
Mark Beamer Jr 1b1d7f5ec0 Add build argument for docker build.sh file
Update compose to use v0.38.6 of lbrynet
2019-08-13 23:03:37 -04:00
Niko Storni adeabc97f8 Merge branch 'setchannelcerts' 2019-08-14 04:30:21 +02:00
Mark Beamer Jr d73c6945dd Fix travis.yml non interactive 2019-08-14 04:22:59 +02:00
Mark Beamer Jr 7a76a4b79d Upgrade to 1.0.17 of lbry.go 2019-08-14 04:22:59 +02:00
Mark Beamer Jr cd57e1ab08 Always send channel export 2019-08-14 04:22:59 +02:00
Mark Beamer Jr b90e367e82 Addressed comments 2019-08-14 04:22:59 +02:00
Mark Beamer Jr e477759395 change to master tag of internal-apis docker image 2019-08-14 04:22:59 +02:00
Mark Beamer Jr 6438048f40 Add CleanOnStartup for local testing and e2e testing
Update methods to be refactored into util package to be used in many locations.
Add end to end test of ytsync for CI
2019-08-14 04:22:59 +02:00
Mark Beamer Jr 73937a541b Add var/tmp volume for ytsync tmp files communicated between lbrynet and ytsync
Add reference to wallet dir env var for when the wallet file is uploaded so it is cleared properly when testing locally
2019-08-14 04:22:59 +02:00
Mark Beamer Jr 517c2c56a3 Add disabling of blob uploading for local testing until reflector docker image can be created( not urgent now ).
Add ability to generate blocks when in regtest mode and using docker.
Add refactor of environment variable accessing.
2019-08-14 04:22:59 +02:00
Mark Beamer Jr c84e1e6fb9 Fixed bugs around sending channel cert and enabling local testing with docker. 2019-08-14 04:22:59 +02:00
Mark Beamer Jr f26712b022 Added e2e test docker-compose and supporting work so that ytsync can run independently of the mainnet for testing/CI 2019-08-14 04:22:59 +02:00
Mark Beamer Jr 5e56ac0516 Add set channel cert on new channels. 2019-08-14 04:22:59 +02:00
Niko Storni 0308da2e4c add timeout to daemon startup 2019-08-13 23:05:09 +02:00
Niko Storni df63b91f61 update reflector libs 2019-08-06 20:57:36 +02:00
Niko Storni 03161a5a02 prefer 1080p videos 2019-08-05 20:08:46 +02:00
Niko Storni 6dc2c39ee6 limit description length
rather than limiting lines count
2019-08-02 10:01:33 -04:00
Niko Storni 87211e802d reduce tags length 2019-08-02 09:05:24 -04:00
Niko Storni 3ba0c97634 fix flags 2019-07-25 18:33:05 -04:00
Niko Storni 9d65fce6bc do not apply limits if disabled 2019-07-25 18:22:56 -04:00
Niko Storni d5ed4ce753 facilitate streaming by moving the moov atom 2019-07-25 18:15:14 -04:00
Niko Storni 0aedf80386 limit throttling to only queued channels 2019-07-22 16:24:13 -04:00
Niko Storni c193dae95a temporary throttling of channels to sync for tom to review them 2019-07-22 16:16:23 -04:00
Niko Storni 0c47510785 revert quoting paths 2019-07-22 02:51:13 +02:00
Niko Storni 8f556a86c8 fix missing fragments for good
never retry them again
2019-07-22 02:45:38 +02:00
Niko Storni 64ab2490fe fix bugs
improve debugging
2019-07-22 02:27:14 +02:00
Niko Storni 1760935edf don't retry missing fragments
lower syncing quality for now
2019-07-22 02:09:18 +02:00
Niko Storni 60f2585f33 improve throttling
refactor slack wrapper
cleanup dependencies
2019-07-15 16:16:02 -04:00
Niko Storni df08d42d9b improve ip re-selection 2019-07-13 12:15:19 +02:00
Niko Storni 1b55033bb8 don't retry big videos 2019-07-13 11:40:50 +02:00
Niko Storni 1331e5769f actually use the functions I create... 2019-07-13 11:33:40 +02:00
Niko Storni 43f745ae20 improve ip shuffling (prevent ips in use from being dealt) 2019-07-12 23:54:48 +02:00
Niko Storni 8c2a8262e6 fix bugs 2019-07-12 23:20:01 +02:00
Niko Storni 6f486717da fix stop pattern
fix noretry string
fix youtube-dl param
2019-07-12 22:58:34 +02:00
Niko Storni a3fcd67611 better handle interruptions by user
refactor IP throttling in its own package
2019-07-12 21:32:49 +02:00
Niko Storni ae1ffb60c5 add ip throttling and better pooling
fix nil pointer dereference
2019-07-12 20:42:44 +02:00
Niko Storni fa24d83ae9 add max length filter 2019-07-12 02:39:24 +02:00
Niko Storni 2f11d99a97 stop on rare bug/failure 2019-07-12 02:21:27 +02:00
Niko Storni bab01a6354 prefer IPv4 over IPv6
youtube sucks and they ban /64 subnet blocks on IPv6
2019-07-12 01:54:34 +02:00
Niko Storni 89fe1e99d6 improve ipv6 validation
fix audio merge issue
2019-07-12 01:28:00 +02:00
Niko Storni ec81af0857 fix download filters 2019-07-12 01:04:45 +02:00
Niko Storni 51cb7f5e34 improve youtube antithrottle
fix logging
2019-07-11 19:14:15 +02:00
Niko Storni 8ad2acc936 remove original downloader
add locking to IP selection to avoid concurrent access
2019-07-11 17:32:46 +02:00
Niko Storni b8caf3f82b add IPv6 source pooling to bypass youtube throttling 2019-07-11 16:22:58 +02:00
Niko Storni b34d328cba halt ytsync when youtube throttles us 2019-07-10 22:47:01 +02:00
Niko Storni a0dcb2216d close db handles 2019-07-10 22:23:57 +02:00
Niko Storni 9d6a7e03cd update prism dependency 2019-07-10 21:07:48 +02:00
Niko Storni e44e272689 adjust config name 2019-07-10 18:04:05 +02:00
Niko Storni 87f7d4c9f9 recreate directory when emptied 2019-07-10 17:42:28 +02:00
Niko Storni 173e62d286 add check to avoid deleting unreflected blobs 2019-07-10 17:39:58 +02:00
Niko Storni 013760d48d delete blobs after reflection
remove 15 seconds wait after sync
2019-07-10 17:27:46 +02:00
Niko Storni 5dbad2d2e4 add automatic blob reflection
retry videos that fail due to tx issues
2019-07-10 15:46:54 +02:00
Niko Storni 545ab07013 fail channels that should fail 2019-07-05 17:35:20 +02:00
Niko Storni c402a1cdef add locking to map reads 2019-06-26 14:40:40 -04:00
Niko Storni a4c6b2da30 actually count UTXOs 2019-06-25 22:28:43 -04:00
Niko Storni 7f48220a80 remove dist folder... 2019-06-26 03:48:44 +02:00
Niko Storni 9e49110295 fix hash 2019-06-26 03:37:52 +02:00
Niko Storni 4506bdb638 fix bug in utxo counting 2019-06-25 21:12:15 -04:00
Niko Storni a8551bc9ee fix UTXO waiting logic 2019-06-25 20:45:27 -04:00
Niko Storni b5c9806dd0 improve space management 2019-06-24 20:43:50 -04:00
Niko Storni 8c349cf7ac fix bug in queues 2019-06-24 04:14:27 -04:00
Niko Storni f3f40c41ef fix refill balance 2019-06-22 06:08:11 -04:00
Niko Storni 05b65c4cb5 add noretry failure 2019-06-21 16:41:46 +02:00
Niko Storni 69ad13848b fix utxo count
fix one bug
add content type to uploaded thumbs
2019-06-20 21:45:17 +02:00
Niko Storni e496ad670a limit tag length 2019-06-20 17:43:46 +02:00
Niko 9d8eb32f9f Merge pull request #28 from lbryio/metadata_upgrade
Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata
2019-06-13 20:14:13 +02:00
Niko Storni 6668840b11 fix description not containing youtube link
add workaround to SDK bug not spending the right UTXOs
fix bug in assignments
2019-06-13 19:33:58 +02:00
Niko Storni e893f68bc1 fix queue pulling
fix address reuse
2019-06-12 22:42:42 +02:00
Niko Storni 8da631eac3 add a limit to UTXOs generation
fix broadcastFee
fix spelling
2019-06-12 05:33:13 +02:00
Niko Storni 1729af0ae5 fix UTXOs management while publishing
fix account_set
improve logging
2019-06-12 05:25:01 +02:00
Niko Storni dce71d2451 add adjustments to video tracking
reduce minimum refill amount (min account balance supersedes)
revert account_fund change as --everything isn't supported with outputs
fix bugs
2019-06-12 04:44:56 +02:00
Niko Storni f4e75cf221 rewrite account funding algorithm
lock publishing when UTXOs management is in progress
spend everything when funding UTXOs
update lbry.go
2019-06-12 03:17:59 +02:00
Niko Storni ae3dad944d add blocking support for publishing and upgrading
add channel wide mappings
2019-06-11 19:18:39 +02:00
Niko Storni f7f0231c80 add VideoUpgradeFailed staus 2019-06-10 21:59:42 +02:00
Niko Storni 0c42b96332 remove unused script
update readme
2019-06-10 21:43:10 +02:00
Niko Storni 52cd483f47 add single channelwide tag
fix bug
2019-06-10 21:37:13 +02:00
Niko Storni 79dd6d1458 add better handling for database mismatches
improve UTXOs management
fix bugs
2019-06-10 02:23:07 +02:00
Niko Storni 10ad46f464 various bug fixes 2019-06-06 23:25:31 +02:00
71 changed files with 7036 additions and 1634 deletions

No files matched your search

+6 -1
View File
@@ -1 +1,6 @@
bin/
bin/
e2e/persist
e2e/supporty/supporty
.env
blobsfiles
ytsync_docker
+8 -8
View File
@@ -9,12 +9,13 @@ builds:
- amd64
ldflags:
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
archive:
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
replacements:
linux: Linux
amd64: x86_64
format: zip
archives:
- id: zip
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
replacements:
linux: Linux
amd64: x86_64
format: zip
checksum:
name_template: 'checksums.txt'
snapshot:
@@ -24,5 +25,4 @@ changelog:
filters:
exclude:
- '^docs:'
- '^test:'
- '^test:'
+31 -6
View File
@@ -1,15 +1,40 @@
os: linux
dist: xenial
dist: bionic
language: go
go_import_path: github.com/lbryio/ytsync
go:
- 1.11.5
- master
- 1.16.3
install: true
cache:
directories:
- $HOME/.cache/go-build
- $HOME/gopath/pkg/mod
services:
- docker
addons:
apt:
update: true
packages:
- ffmpeg
- tree
- python3-pip
before_script:
- sudo pip3 install -U yt-dlp
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
script:
#- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
- make
deploy:
provider: script
skip_cleanup: true
@@ -17,4 +42,4 @@ deploy:
file: bin/ytsync
on:
repo: lbryio/ytsync
tags: true
tags: true
+15
View File
@@ -0,0 +1,15 @@
The MIT License (MIT)
Copyright (c) 2017-2020 LBRY Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+1 -1
View File
@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
+3 -1
View File
@@ -55,7 +55,7 @@ Usage:
Flags:
--after int Specify from when to pull jobs [Unix time](Default: 0)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1582834707)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
--channelID string If specified, only this channel will be synced.
--concurrent-jobs int how many jobs to process concurrently (default 1)
-h, --help help for ytsync
@@ -63,12 +63,14 @@ Flags:
--max-length float Maximum video length to process (in hours) (default 2)
--max-size int Maximum video size to process (in MB) (default 2048)
--max-tries int Number of times to try a publish that fails (default 3)
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
--run-once Whether the process should be stopped after one cycle or not
--skip-space-check Do not perform free space check on startup
--status string Specify which queue to pull from. Overrides --update
--stop-on-error If a publish fails, stop all publishing and exit
--takeover-existing-channel If channel exists and we don't own it, take over the channel
--update Update previously synced channels instead of syncing new ones
--upgrade-metadata Upgrade videos if they're on the old metadata version
--videos-limit int how many videos to process per channel (default 1000)
```
+93
View File
@@ -0,0 +1,93 @@
package blobs_reflector
import (
"encoding/json"
"io/ioutil"
"os"
"os/user"
"path/filepath"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/reflector.go/cmd"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/sirupsen/logrus"
"github.com/lbryio/ytsync/v5/util"
)
var dbHandle *db.SQL
func ReflectAndClean() error {
err := reflectBlobs()
if err != nil {
return err
}
return util.CleanupLbrynet()
}
func loadConfig(path string) (cmd.Config, error) {
var c cmd.Config
raw, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, err
}
err = json.Unmarshal(raw, &c)
return c, err
}
func reflectBlobs() error {
if util.IsBlobReflectionOff() {
return nil
}
logrus.Infoln("reflecting blobs...")
//make sure lbrynet is off
running, err := util.IsLbrynetRunning()
if err != nil {
return err
}
if running {
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
}
logrus.SetLevel(logrus.InfoLevel)
defer logrus.SetLevel(logrus.DebugLevel)
ex, err := os.Executable()
if err != nil {
return errors.Err(err)
}
exPath := filepath.Dir(ex)
config, err := loadConfig(exPath + "/prism_config.json")
if err != nil {
return errors.Err(err)
}
if dbHandle == nil {
dbHandle = new(db.SQL)
err = dbHandle.Connect(config.DBConn)
if err != nil {
return errors.Err(err)
}
}
st := store.NewDBBackedStore(store.NewS3Store(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName), dbHandle, false)
uploadWorkers := 10
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
usr, err := user.Current()
if err != nil {
return errors.Err(err)
}
blobsDir := usr.HomeDir + "/.lbrynet/blobfiles/"
err = uploader.Upload(blobsDir)
if err != nil {
return errors.Err(err)
}
if uploader.GetSummary().Err > 0 {
return errors.Err("not al blobs were reflected. Errors: %d", uploader.GetSummary().Err)
}
return nil
}
+7
View File
@@ -0,0 +1,7 @@
package cmd
import "github.com/lbryio/ytsync/v5/local"
func init() {
local.AddCommand(rootCmd)
}
+166
View File
@@ -0,0 +1,166 @@
package cmd
import (
"os"
"time"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
const defaultMaxTries = 3
var (
cliFlags shared.SyncFlags
maxVideoLength int
)
var rootCmd = &cobra.Command{
Use: "ytsync",
Short: "Publish youtube channels into LBRY network automatically.",
Run: ytSync,
Args: cobra.RangeArgs(0, 0),
}
func init() {
rootCmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
rootCmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
rootCmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
rootCmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
rootCmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
rootCmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
rootCmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
rootCmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
rootCmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
rootCmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
rootCmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
rootCmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
rootCmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
rootCmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
rootCmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
rootCmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
rootCmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
rootCmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
rootCmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
}
func Execute() {
err := rootCmd.Execute()
if err != nil {
log.Errorln(err)
os.Exit(1)
}
}
func ytSync(cmd *cobra.Command, args []string) {
var hostname string
slackToken := os.Getenv("SLACK_TOKEN")
if slackToken == "" {
log.Error("A slack token was not present in env vars! Slack messages disabled!")
} else {
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync-unknown"
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
}
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
return
}
if cliFlags.MaxTries < 1 {
log.Errorln("setting --max-tries less than 1 doesn't make sense")
return
}
if cliFlags.Limit < 0 {
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
return
}
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
apiURL := os.Getenv("LBRY_WEB_API")
apiToken := os.Getenv("LBRY_API_TOKEN")
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
awsS3ID := os.Getenv("AWS_S3_ID")
awsS3Secret := os.Getenv("AWS_S3_SECRET")
awsS3Region := os.Getenv("AWS_S3_REGION")
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
if apiURL == "" {
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
return
}
if apiToken == "" {
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
return
}
if youtubeAPIKey == "" {
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
return
}
if awsS3ID == "" {
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
return
}
if awsS3Secret == "" {
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
return
}
if awsS3Region == "" {
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
return
}
if awsS3Bucket == "" {
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
return
}
if lbrycrdDsn == "" {
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
}
blobsDir := ytUtils.GetBlobsDir()
apiConfig := &sdk.APIConfig{
YoutubeAPIKey: youtubeAPIKey,
ApiURL: apiURL,
ApiToken: apiToken,
HostName: hostname,
}
awsConfig := &shared.AwsConfigs{
AwsS3ID: awsS3ID,
AwsS3Secret: awsS3Secret,
AwsS3Region: awsS3Region,
AwsS3Bucket: awsS3Bucket,
}
sm := manager.NewSyncManager(
cliFlags,
blobsDir,
lbrycrdDsn,
awsConfig,
apiConfig,
)
err := sm.Start()
if err != nil {
ytUtils.SendErrorToSlack(errors.FullTrace(err))
}
ytUtils.SendInfoToSlack("Syncing process terminated!")
}
+394
View File
@@ -0,0 +1,394 @@
package downloader
import (
"encoding/json"
"fmt"
"io/ioutil"
"math"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"strings"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
util2 "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/sirupsen/logrus"
)
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
ids, err := run(channelName, args, stopChan, pool)
if err != nil {
return nil, errors.Err(err)
}
videoIDs := make([]string, 0, maxVideos)
for i, v := range ids {
logrus.Debugf("%d - video id %s", i, v)
if i >= maxVideos {
break
}
videoIDs = append(videoIDs, v)
}
return videoIDs, nil
}
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
args := []string{
"--skip-download",
"--write-info-json",
fmt.Sprintf("https://www.youtube.com/watch?v=%s", videoID),
"--cookies",
"cookies.txt",
"-o",
path.Join(util2.GetVideoMetadataDir(), videoID),
}
_, err := run(videoID, args, stopChan, pool)
if err != nil {
return nil, errors.Err(err)
}
f, err := os.Open(path.Join(util2.GetVideoMetadataDir(), videoID+".info.json"))
if err != nil {
return nil, errors.Err(err)
}
// defer the closing of our jsonFile so that we can parse it later on
defer f.Close()
// read our opened jsonFile as a byte array.
byteValue, _ := ioutil.ReadAll(f)
var video *ytdl.YtdlVideo
err = json.Unmarshal(byteValue, &video)
if err != nil {
return nil, errors.Err(err)
}
// now get an accurate time
const maxTries = 5
tries := 0
GetTime:
tries++
t, err := getUploadTime(config, videoID, ip, video.UploadDate)
if err != nil {
//slack(":warning: Upload time error: %v", err)
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
err := triggerScrape(videoID, ip)
if err == nil {
time.Sleep(2 * time.Second) // let them scrape it
goto GetTime
} else {
//slack("triggering scrape returned error: %v", err)
}
} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
if t == "" {
return nil, errors.Err(err)
} else {
t = "" //TODO: get rid of the other piece below?
}
}
// do fallback below
}
//slack("After all that, upload time for %s is %s", videoID, t)
if t != "" {
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
if err != nil {
return nil, errors.Err(err)
}
//slack(":exclamation: Got an accurate time for %s", videoID)
video.UploadDateForReal = parsed
} else { //TODO: this is the piece that isn't needed!
slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
// fall back to UploadDate from youtube-dl
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
if err != nil {
return nil, err
}
}
return video, nil
}
var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
var errUploadTimeEmpty = errors.Base("upload time is empty")
var errStatusParse = errors.Base("could not parse status, got number, need string")
var errConnectionIssue = errors.Base("there was a connection issue with the api")
func slack(format string, a ...interface{}) {
fmt.Printf(format+"\n", a...)
util.SendToSlack(format, a...)
}
func triggerScrape(videoID string, ip *net.TCPAddr) error {
//slack("Triggering scrape for %s", videoID)
u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
q := u.Query()
q.Set("twtimeline", "0")
q.Set("url", "https://www.youtube.com/watch?v="+videoID)
u.RawQuery = q.Encode()
//slack("GET %s", u.String())
client := getClient(ip)
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return errors.Err(err)
}
req.Header.Set("User-Agent", ChromeUA)
res, err := client.Do(req)
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
var response struct {
Message string `json:"message"`
Status string `json:"status"`
VideoURL string `json:"video_url"`
}
err = json.NewDecoder(res.Body).Decode(&response)
if err != nil {
if strings.Contains(err.Error(), "cannot unmarshal number") {
return errors.Err(errStatusParse)
}
if strings.Contains(err.Error(), "no route to host") {
return errors.Err(errConnectionIssue)
}
return errors.Err(err)
}
switch response.Status {
case "removed_video":
return errors.Err("video previously removed from service")
case "no_video":
return errors.Err("they say 'video cannot be found'. wtf?")
default:
spew.Dump(response)
}
return nil
//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
}
func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
//slack("Getting upload time for %s", videoID)
release, err := config.GetReleasedDate(videoID)
if err != nil {
logrus.Error(err)
}
ytdlUploadDate, err := time.Parse("20060102", uploadDate)
if err != nil {
logrus.Error(err)
}
if release != nil {
//const sqlTimeFormat = "2006-01-02 15:04:05"
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
if err == nil {
hoursDiff := math.Abs(sqlTime.Sub(ytdlUploadDate).Hours())
if hoursDiff > 48 {
logrus.Infof("upload day from APIs differs from the ytdl one by more than 2 days.")
} else {
return sqlTime.Format(releaseTimeFormat), nil
}
} else {
logrus.Error(err)
}
}
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
return ytdlUploadDate.Format(releaseTimeFormat), nil
}
client := getClient(ip)
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
req.Header.Set("User-Agent", ChromeUA)
res, err := client.Do(req)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
defer res.Body.Close()
var uploadTime struct {
Time string `json:"video_upload_time"`
Message string `json:"message"`
Status string `json:"status"`
}
err = json.NewDecoder(res.Body).Decode(&uploadTime)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
if uploadTime.Status == "ERROR1" {
return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
}
if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
}
if uploadTime.Time == "" {
return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
}
return uploadTime.Time, nil
}
func getClient(ip *net.TCPAddr) *http.Client {
if ip == nil {
return http.DefaultClient
}
return &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
LocalAddr: ip,
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
}
}
const (
GoogleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
maxAttempts = 3
extractionError = "YouTube said: Unable to extract video data"
throttledError = "HTTP Error 429"
AlternateThrottledError = "returned non-zero exit status 8"
youtubeDlError = "exit status 1"
videoPremiereError = "Premieres in"
liveEventError = "This live event will begin in"
)
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
var useragent []string
var lastError error
for attempts := 0; attempts < maxAttempts; attempts++ {
sourceAddress, err := getIPFromPool(use, stopChan, pool)
if err != nil {
return nil, err
}
argsForCommand := append(args, "--source-address", sourceAddress)
argsForCommand = append(argsForCommand, useragent...)
binary := "yt-dlp"
cmd := exec.Command(binary, argsForCommand...)
res, err := runCmd(cmd, stopChan)
pool.ReleaseIP(sourceAddress)
if err == nil {
return res, nil
}
lastError = err
if strings.Contains(err.Error(), youtubeDlError) {
if util.SubstringInSlice(err.Error(), shared.ErrorsNoRetry) {
break
}
if strings.Contains(err.Error(), extractionError) {
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
useragent = nextUA(useragent)
}
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
pool.SetThrottled(sourceAddress)
//we don't want throttle errors to count toward the max retries
attempts--
}
}
}
return nil, lastError
}
func nextUA(current []string) []string {
if len(current) == 0 {
return []string{"--user-agent", GoogleBotUA}
}
return []string{"--user-agent", ChromeUA}
}
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
logrus.Infof("running yt-dlp cmd: %s", strings.Join(cmd.Args, " "))
var err error
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, errors.Err(err)
}
err = cmd.Start()
if err != nil {
return nil, errors.Err(err)
}
outLog, err := ioutil.ReadAll(stdout)
if err != nil {
return nil, errors.Err(err)
}
errorLog, err := ioutil.ReadAll(stderr)
if err != nil {
return nil, errors.Err(err)
}
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
select {
case <-stopChan:
err := cmd.Process.Kill()
if err != nil {
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
}
return nil, errors.Err("interrupted by user")
case err := <-done:
if err != nil {
return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
}
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
}
}
func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
for {
sourceAddress, err = pool.GetIP(use)
if err != nil {
if errors.Is(err, ip_manager.ErrAllThrottled) {
select {
case <-stopChan:
return "", errors.Err("interrupted by user")
default:
time.Sleep(ip_manager.IPCooldownPeriod)
continue
}
} else {
return "", err
}
}
break
}
return
}
+42
View File
@@ -0,0 +1,42 @@
package downloader
import (
"testing"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestGetPlaylistVideoIDs(t *testing.T) {
videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
if err != nil {
logrus.Error(err)
}
for _, id := range videoIDs {
println(id)
}
}
func TestGetVideoInformation(t *testing.T) {
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
if err != nil {
logrus.Error(err)
}
if video != nil {
logrus.Info(video.ID)
}
}
func Test_getUploadTime(t *testing.T) {
configs := sdk.APIConfig{
YoutubeAPIKey: "",
ApiURL: "https://api.lbry.com",
ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
HostName: "test",
}
got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
assert.NoError(t, err)
t.Log(got)
}
+150
View File
@@ -0,0 +1,150 @@
package ytdl
import (
"time"
)
type YtdlVideo struct {
UploadDate string `json:"upload_date"`
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
Extractor string `json:"extractor"`
Series interface{} `json:"series"`
Format string `json:"format"`
Vbr interface{} `json:"vbr"`
Chapters interface{} `json:"chapters"`
Height int `json:"height"`
LikeCount interface{} `json:"like_count"`
Duration int `json:"duration"`
Fulltitle string `json:"fulltitle"`
PlaylistIndex interface{} `json:"playlist_index"`
Album interface{} `json:"album"`
ViewCount int `json:"view_count"`
Playlist interface{} `json:"playlist"`
Title string `json:"title"`
Filename string `json:"_filename"`
Creator interface{} `json:"creator"`
Ext string `json:"ext"`
ID string `json:"id"`
DislikeCount interface{} `json:"dislike_count"`
AverageRating float64 `json:"average_rating"`
Abr float64 `json:"abr"`
UploaderURL string `json:"uploader_url"`
Categories []string `json:"categories"`
Fps float64 `json:"fps"`
StretchedRatio interface{} `json:"stretched_ratio"`
SeasonNumber interface{} `json:"season_number"`
Annotations interface{} `json:"annotations"`
WebpageURLBasename string `json:"webpage_url_basename"`
Acodec string `json:"acodec"`
DisplayID string `json:"display_id"`
//RequestedFormats []RequestedFormat `json:"requested_formats"`
//AutomaticCaptions struct{} `json:"automatic_captions"`
Description string `json:"description"`
Tags []string `json:"tags"`
Track interface{} `json:"track"`
RequestedSubtitles interface{} `json:"requested_subtitles"`
StartTime interface{} `json:"start_time"`
Uploader string `json:"uploader"`
ExtractorKey string `json:"extractor_key"`
FormatID string `json:"format_id"`
EpisodeNumber interface{} `json:"episode_number"`
UploaderID string `json:"uploader_id"`
//Subtitles struct{} `json:"subtitles"`
ReleaseYear interface{} `json:"release_year"`
Thumbnails []Thumbnail `json:"thumbnails"`
License interface{} `json:"license"`
Artist interface{} `json:"artist"`
AgeLimit int `json:"age_limit"`
ReleaseDate interface{} `json:"release_date"`
AltTitle interface{} `json:"alt_title"`
Thumbnail string `json:"thumbnail"`
ChannelID string `json:"channel_id"`
IsLive interface{} `json:"is_live"`
Width int `json:"width"`
EndTime interface{} `json:"end_time"`
WebpageURL string `json:"webpage_url"`
Formats []Format `json:"formats"`
ChannelURL string `json:"channel_url"`
Resolution interface{} `json:"resolution"`
Vcodec string `json:"vcodec"`
}
type RequestedFormat struct {
Asr interface{} `json:"asr"`
Tbr float64 `json:"tbr"`
Container string `json:"container"`
Language interface{} `json:"language"`
Format string `json:"format"`
URL string `json:"url"`
Vcodec string `json:"vcodec"`
FormatNote string `json:"format_note"`
Height int `json:"height"`
Width int `json:"width"`
Ext string `json:"ext"`
FragmentBaseURL string `json:"fragment_base_url"`
Filesize interface{} `json:"filesize"`
Fps float64 `json:"fps"`
ManifestURL string `json:"manifest_url"`
Protocol string `json:"protocol"`
FormatID string `json:"format_id"`
HTTPHeaders struct {
AcceptCharset string `json:"Accept-Charset"`
AcceptLanguage string `json:"Accept-Language"`
AcceptEncoding string `json:"Accept-Encoding"`
Accept string `json:"Accept"`
UserAgent string `json:"User-Agent"`
} `json:"http_headers"`
Fragments []struct {
Path string `json:"path"`
Duration float64 `json:"duration,omitempty"`
} `json:"fragments"`
Acodec string `json:"acodec"`
Abr int `json:"abr,omitempty"`
}
type Format struct {
Asr int `json:"asr"`
Filesize int `json:"filesize"`
FormatID string `json:"format_id"`
FormatNote string `json:"format_note"`
Fps interface{} `json:"fps"`
Height interface{} `json:"height"`
Quality int `json:"quality"`
Tbr float64 `json:"tbr"`
URL string `json:"url"`
Width interface{} `json:"width"`
Ext string `json:"ext"`
Vcodec string `json:"vcodec"`
Acodec string `json:"acodec"`
Abr float64 `json:"abr,omitempty"`
DownloaderOptions struct {
HTTPChunkSize int `json:"http_chunk_size"`
} `json:"downloader_options,omitempty"`
Container string `json:"container,omitempty"`
Format string `json:"format"`
Protocol string `json:"protocol"`
HTTPHeaders struct {
UserAgent string `json:"User-Agent"`
AcceptCharset string `json:"Accept-Charset"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Vbr float64 `json:"vbr,omitempty"`
}
type Thumbnail struct {
URL string `json:"url"`
Width int `json:"width"`
Resolution string `json:"resolution"`
ID string `json:"id"`
Height int `json:"height"`
}
type HTTPHeaders struct {
AcceptCharset string `json:"Accept-Charset"`
AcceptLanguage string `json:"Accept-Language"`
AcceptEncoding string `json:"Accept-Encoding"`
Accept string `json:"Accept"`
UserAgent string `json:"User-Agent"`
}
+28
View File
@@ -0,0 +1,28 @@
version: '3.4'
services:
###########
## MYSQL ##
###########
mysql:
image: mysql:5.7.23
restart: "no"
ports:
- 3306:3306
volumes:
- "../persist/chainquery/db:/var/lib/mysql"
## This one may need to be tweaked based on where you run this docker-compose from.
- "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
################
## Chainquery ##
################
chainquery:
image: lbry/chainquery:v1.8.1
restart: "no"
ports:
- 6300:6300
depends_on:
- mysql
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
volumes:
- "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml"
+33
View File
@@ -0,0 +1,33 @@
## Get the latest source and extract it for the app container.
## Design choices, two RUN layers intended to keep builds faster, the zipped
FROM ubuntu:18.04 as prep
LABEL MAINTAINER="leopere [at] nixc [dot] us"
RUN apt-get update && \
apt-get -y install unzip curl telnet wait-for-it && \
apt-get autoclean -y && \
rm -rf /var/lib/apt/lists/*
WORKDIR /
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
COPY ./start.sh start
COPY ./healthcheck.sh healthcheck
ARG VERSION="master"
RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \
chmod +x /chainquery
FROM ubuntu:18.04 as app
RUN apt-get update && \
apt-get -y install telnet wait-for-it && \
apt-get autoclean -y && \
rm -rf /var/lib/apt/lists/*
ARG VERSION="master"
ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig
RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \
chown -R chainquery:chainquery /etc/lbry
COPY --from=prep ./healthcheck /chainquery /start /usr/bin/
HEALTHCHECK --interval=1m --timeout=30s \
CMD healthcheck
EXPOSE 6300
USER chainquery
STOPSIGNAL SIGINT
CMD ["start"]
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
if [ $# -eq 0 ]
then
echo "No docker tag argument supplied. Use './build.sh <tag>'"
exit 1
fi
docker build --no-cache --build-arg VERSION=$1 --tag lbry/chainquery:$1 .
docker push lbry/chainquery:$1
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
curl --fail http://localhost:6300/api/status || exit 1
+9
View File
@@ -0,0 +1,9 @@
# Default Homebrew MySQL server config
[mysqld]
# Only allow connections from localhost
innodb_log_file_size=5G
key_buffer_size=1G
innodb_flush_log_at_trx_commit = 0
innodb_autoinc_lock_mode=2
innodb_buffer_pool_size=1G
innodb_log_buffer_size=1G
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
## Config setup
## Setup Values
DEBUGMODE=$(echo "debugmode=$DEBUGMODE")
LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"")
MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
## Setup Defaults
DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false'
LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"'
MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
## Add setup value variable name to this list to get processed on container start
CONFIG_SETTINGS=(
DEBUGMODE
LBRYCRDURL
MYSQLDSN
APIMYSQLDSN
)
function set_configs() {
## Set configs on container start if not already set.
for i in "${!CONFIG_SETTINGS[@]}"; do
## Indirect references http://tldp.org/LDP/abs/html/ivr.html
eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT"
eval TO_STRING=\$${CONFIG_SETTINGS[$i]}
## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts.
sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml
done
echo "Reading config for debugging."
cat /etc/lbry/chainqueryconfig.toml
}
if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then
echo "[INFO]: Did not find chainqueryconfig.toml"
echo " Installing default and configuring with provided environment variables if any."
## Install fresh copy of config file.
echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml"
cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml
ls -lAh /etc/lbry/
set_configs
else
echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry"
fi
## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
chainquery serve --configpath "/etc/lbry/"
+99
View File
@@ -0,0 +1,99 @@
#Debug mode outputs specific information to the console
#DEFAULT: false
#debugmode=
#DebugQueryMode outputs SQL Boiler queries to the console.
#DEFAULT: false
#debugquerymode=
#LBRYcrd URL is required for chainquery to query the blockchain
#DEFAULT: "rpc://lbry:lbry@localhost:9245"
lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
#MySQL DSN is required for chainquery to store information.
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
#API MySQL DSN is required for chainquery to expose a SQL query service
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
#API Host and Port is required for the API Server to bind and listen on.
#DEFAULT: "0.0.0.0:6300"
#apihostport=
#Profile mode enables and disables the reporting of a profile for chainquery
#DEFAULT: false
#profilemode=
#Daemon mode tells chainquery how hard it should work catch up processing the blockchain
#deamonmode=0 #BeastMode it continuously process block after block until caughtup.
#daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms
#daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay')
#daemonmode=3 #DaemonMode it will process a block every iteration of the daemon.
#DEFAULT: 0
#deamonmode=
#Default client timeout is for communication with the api of chainquery
#DEFAULT: 20 #Measured in seconds
#defaultclienttimeout=
#Processing delay is used to determine how frequently chainquery should process a block
# It is only used if Daemon mode is set to delay mode
#DEFAULT: 100 #Measured in milliseconds
#processingdelay=
#Daemon delay is the frequency at which chainquery checks for work to do.
#DEFAULT: 1 #Measured in seconds
#daemondelay=
#Profiling options - will output the time take for certain opertions related to the below category
#DEFAULT: false (for all 3 params)
#daemonprofile=
#lbrycrdprofile=
#mysqlprofile=
#Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel.
#DEFAULT: ""
#slackhookurl=
#Slack Channel is the channel that you want the messages to appear. Works together with the hook url.
#DEFAULT: ""
#slackchannel=
#Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below
# it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5
#DEFAULT: 0
#slackloglevel=
#The command that should be executed to trigger a self update of the software. For linux, for example, `<yourscript>.sh`
#DEFAULT: ""
#autoupdatecommand=
#Twilio service of chainquery to send specifically important information to key users of the Chainquery install.
#DEFAULT:
##twiliosid=""
##twilioauthtoken=""
##smsrecipients=["",""]
##smsfromphonenumber=""
#twiliosid=
#twilioauthtoken=
#smsrecipients=
#smsfromphonenumber=
#API Keys - Disallowed by default unless keys are entered.
#DEFAULT: []
#apikeys=
#Max Failures - Specifies the number of failures that can happen in processing a transaction. This is for parallel
#transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its
#source output to spend is not already processed.
#DEFAULT: 1000
#maxfailures=
#Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are
#lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest.
#DEFAULT: "lbrycrd_main"
blockchainname="lbrycrd_regtest"
+10
View File
@@ -0,0 +1,10 @@
blockchain_name: lbrycrd_regtest
lbryum_servers:
- walletserver:50001
reflect_streams: false
save_blobs: true
save_files: false
share_usage_data: false
tcp_port: 3333
udp_port: 4444
use_upnp: false
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -e
#Add a ytsync user
ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
#Insert an auth token for the user to be used by ytsync
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Give priveledges to ytsync user
ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
#Add youtuber to sync
ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
#Insert an auth token for the youtuber to be used by ytsync
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"
+140
View File
@@ -0,0 +1,140 @@
version: "3.5"
services:
#############
## Lbrycrd ##
#############
lbrycrd:
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
restart: "no"
ports:
- "15201:29246"
- "15200:29245"
expose:
- "29246"
- "29245"
## host volumes for persistent data such as wallet private keys.
volumes:
- "./persist:/data"
environment:
- RUN_MODE=regtest
###################
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:latest-release
restart: always
environment:
- DB_DIRECTORY=/database
- MAX_SEND=1000000000000000000000
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245
- MAX_SUBS=1000000000000
- BANDWIDTH_LIMIT=80000000000
- SESSION_TIMEOUT=10000000000000000000000000
- TCP_PORT=50001
- ELASTIC_HOST=es01
ports:
- "15300:50001"
expose:
- "50001"
depends_on:
- lbrycrd
- es01
ulimits:
nofile:
soft: 90000
hard: 90000
#command: lbry.wallet.server.coin.LBC
command: lbry.wallet.server.coin.LBCRegTest
#############
## elasticsearch ##
#############
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
container_name: es01
environment:
- node.name=es01
- discovery.type=single-node
- indices.query.bool.max_clause_count=8196
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
ulimits:
memlock:
soft: -1
hard: -1
ports:
- "9200:9200"
expose:
- "9200"
#############
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.99.0
restart: always
ports:
- "15100:5279"
- "15101:5280"
expose:
- "5279"
- "5280"
depends_on:
- walletserver
environment:
- LBRY_STREAMING_SERVER=0.0.0.0:5280
volumes:
- "./persist/.lbrynet:/home/lbrynet"
- ".:/etc/lbry" #Put your daemon_settings.yml here
# /private/var/tmp for OSX and /var/tmp for Linux
- "${LOCAL_TMP_DIR}"
###########
## MySQL ##
###########
mysql:
image: mysql/mysql-server:5.7.33
restart: "no"
ports:
- "15500:3306"
expose:
- "3306"
environment:
- MYSQL_ALLOW_EMPTY_PASSWORD=true
- MYSQL_DATABASE=lbry
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
###################
## Internal APIs ##
###################
internalapis:
image: lbry/internal-apis:master
restart: "no"
ports:
- "15400:8080"
expose:
- "8080"
depends_on:
- mysql
- lbrycrd
environment:
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
- REPLICA_DSN=lbry:lbry@tcp(mysql:3306)/lbry
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
################
## Chainquery ##
################
chainquery:
image: lbry/chainquery:master
restart: "no"
ports:
- 6300:6300
depends_on:
- lbrycrd
- mysql
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
volumes:
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
Executable
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env bash
set -e
#Always compile ytsync
make
#Always compile supporty
cd e2e/supporty && make && cd ../..
#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
touch -a .env && set -o allexport; source ./.env; set +o allexport
echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
# Compose settings - docker only
export SLACK_CHANNEL="ytsync-travis"
export LBRY_API_TOKEN="ytsyntoken"
export LBRY_WEB_API="http://localhost:15400"
export LBRYNET_ADDRESS="http://localhost:15100"
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
export LBRYNET_USE_DOCKER=true
export REFLECT_BLOBS=false
export CLEAN_ON_STARTUP=true
export REGTEST=true
# Local settings
export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
export TMP_DIR="/var/tmp"
export CHAINNAME="lbrycrd_regtest"
export UID
cd ./e2e
docker-compose stop
docker-compose rm -f
echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
docker-compose pull
if [[ -d persist ]]; then rm -rf persist; fi
mkdir -m 0777 -p ./persist
mkdir -m 777 -p ./persist/.walletserver
mkdir -m 777 -p ./persist/.lbrynet
#sudo chown -Rv 999:999 ./persist/.walletserver
#sudo chown -Rv 1000:1000 ./persist/.lbrynet
docker-compose up -d
printf 'waiting for internal apis'
until curl --output /dev/null --silent --head --fail http://localhost:15400; do
printf '.'
sleep 1
done
echo "successfully started..."
channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
channelName=@Alaminemoh11"$(date +%s)"
latestVideoID="ejWF7Jjdgmc"
#Data Setup for test
./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
# Execute the sync test!
./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
# Create Supports for published claim
./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
# Reset status for transfer test
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
# Trigger transfer api
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
# Execute the transfer test!
./../bin/ytsync --channelID $channelToSync --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
echo "Video Status: $videoStatus"
echo "Channel Transfer Status: $channelTransferStatus"
echo "Video Transfer Status: $videoTransferStatus"
echo "Nr Unspent Supports: $nrUnspentSupports"
#docker-compose logs --tail="all" lbrycrd
#docker-compose logs --tail="all" walletserver
#docker-compose logs --tail="all" lbrynet
#docker-compose logs --tail="all" internalapis
exit 1;
else
echo "SUCCESSSSSSSSSSSSS!"
fi;
docker-compose down
+3
View File
@@ -0,0 +1,3 @@
CREATE DATABASE IF NOT EXISTS chainquery;
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
FLUSH PRIVILEGES;
+26
View File
@@ -0,0 +1,26 @@
version: "3"
networks:
lbry-network:
external: true
services:
#############
## Lbrycrd ##
#############
lbrycrd:
image: lbry/lbrycrd:v0.12.4.1
restart: always
networks:
lbry-network:
ipv4_address: 10.6.1.1
ports:
- "15201:29246"
- "15200:29245"
expose:
- "29246"
- "29245"
## host volumes for persistent data such as wallet private keys.
volumes:
- "../persist/data:/data"
environment:
- RUN_MODE=regtest
+37
View File
@@ -0,0 +1,37 @@
FROM ubuntu:18.04 as prep
LABEL MAINTAINER="leopere [at] nixc [dot] us"
## TODO: Implement version pinning. `apt-get install curl=<version>`
RUN apt-get update && \
apt-get -y install unzip curl build-essential && \
apt-get autoclean -y && \
rm -rf /var/lib/apt/lists/*
WORKDIR /
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
COPY ./start.sh start
COPY ./healthcheck.sh healthcheck
COPY ./advance_blocks.sh advance
COPY ./fix-permissions.c fix-permissions.c
## Add lbrycrd - Change the version below to create an image for a different tag/version
ARG VERSION="v0.12.4.1"
RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL
RUN unzip ./lbrycrd-linux.zip && \
gcc fix-permissions.c -o fix-permissions && \
chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
FROM ubuntu:18.04 as app
COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
RUN addgroup --gid 1000 lbrycrd && \
adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
mkdir /etc/lbry && \
chown lbrycrd /etc/lbry && \
chmod a+s /usr/bin/fix-permissions
VOLUME ["/data"]
WORKDIR /data
## TODO: Implement healthcheck.
# HEALTHCHECK ["healthcheck"]
EXPOSE 9246 9245 11337 29245
USER lbrycrd
CMD ["start"]
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
while true; do
lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
sleep 2
done
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
if [ $# -eq 0 ]
then
echo "No docker tag argument supplied. Use './build.sh <tag>'"
exit 1
fi
docker build --build-arg VERSION=$1 --tag lbry/lbrycrd:$1 .
docker push lbry/lbrycrd:$1
+9
View File
@@ -0,0 +1,9 @@
#include <unistd.h>
int main() {
// This program needs to run with setuid == root
// This needs to be in a compiled language because you cannot setuid bash scripts
setuid(0);
execle("/bin/bash", "bash", "-c",
"/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
(char*) NULL, (char*) NULL);
}
+4
View File
@@ -0,0 +1,4 @@
## TODO: Implement a healthcheck for lbrycrd.
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;' http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
## OR
lbrycrd-cli getinfo
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env bash
CONFIG_PATH=/etc/lbry/lbrycrd.conf
function override_config_option() {
# Remove existing config line from a config file
# and replace with environment fed value.
# Does nothing if the variable does not exist.
# var Name of ENV variable
# option Name of config option
# config Path of config file
local var=$1 option=$2 config=$3
if [[ -v $var ]]; then
# Remove the existing config option:
sed -i "/^$option\W*=/d" "$config"
# Add the value from the environment:
echo "$option=${!var}" >> "$config"
fi
}
function set_config() {
if [ -d "$CONFIG_PATH" ]; then
echo "$CONFIG_PATH is a directory when it should be a file."
exit 1
elif [ -f "$CONFIG_PATH" ]; then
echo "Merging the mounted config file with environment variables."
local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
cat $CONFIG_PATH > $MERGED_CONFIG
echo "" >> $MERGED_CONFIG
override_config_option PORT port $MERGED_CONFIG
override_config_option RPC_USER rpcuser $MERGED_CONFIG
override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
override_config_option RPC_PORT rpcport $MERGED_CONFIG
override_config_option RPC_BIND rpcbind $MERGED_CONFIG
# Make the new merged config file the new CONFIG_PATH
# This ensures that the original file the user mounted remains unmodified
CONFIG_PATH=$MERGED_CONFIG
else
echo "Creating a fresh config file from environment variables."
## Set config params
{
echo "port=${PORT=9246}"
echo "rpcuser=${RPC_USER=lbry}"
echo "rpcpassword=${RPC_PASSWORD=lbry}"
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
echo "rpcport=${RPC_PORT=9245}"
echo "rpcbind=${RPC_BIND=0.0.0.0}"
echo "deprecatedrpc=accounts"
echo "deprecatedrpc=validateaddress"
echo "deprecatedrpc=signrawtransaction"
} >> $CONFIG_PATH
fi
echo "Config: "
cat $CONFIG_PATH
}
## Ensure perms are correct prior to running main binary
/usr/bin/fix-permissions
## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
case $RUN_MODE in
default )
set_config
lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
;;
## If it's a first run you need to do a full index including all transactions
## tx index creates an index of every single transaction in the block history if
## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
## This is generally specific to chainquery.
reindex )
## Apply this RUN_MODE in the case you need to update a dataset. NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
set_config
lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
;;
chainquery )
## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
## fresh local dataset use this run mode.
set_config
lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
;;
regtest )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p "$(dirname $CONFIG_PATH)"
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
echo "rpcbind=0.0.0.0" >> $CONFIG_PATH
echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH
echo "regtest=1" >> $CONFIG_PATH
echo "txindex=1" >> $CONFIG_PATH
echo "server=1" >> $CONFIG_PATH
echo "printtoconsole=1" >> $CONFIG_PATH
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
echo "vbparams=segwit:0:999999999999" >> $CONFIG_PATH
echo "addresstype=legacy" >> $CONFIG_PATH
#nohup advance &>/dev/null &
lbrycrdd -conf=$CONFIG_PATH $1
;;
testnet )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p "$(dirname $CONFIG_PATH)"
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
echo "rpcbind=0.0.0.0" >> $CONFIG_PATH
echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH
echo "testnet=1" >> $CONFIG_PATH
echo "txindex=1" >> $CONFIG_PATH
echo "server=1" >> $CONFIG_PATH
echo "printtoconsole=1" >> $CONFIG_PATH
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
#nohup advance &>/dev/null &
lbrycrdd -conf=$CONFIG_PATH $1
;;
* )
echo "Error, you must define a RUN_MODE environment variable."
echo "Available options are testnet, regtest, chainquery, default, and reindex"
;;
esac
+23
View File
@@ -0,0 +1,23 @@
version: "3"
networks:
lbry-network:
external: true
services:
#############
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.99.0
restart: "no"
networks:
lbry-network:
ipv4_address: 10.6.1.3
ports:
- "15100:5279"
- "15101:5280"
environment:
- LBRY_STREAMING_SERVER=0.0.0.0:5280
volumes:
- "../persist/data/.lbrynet:/home/lbrynet"
- "./settings:/etc/lbry" #Put your daemon_settings.yml here
+27
View File
@@ -0,0 +1,27 @@
## This base image is for running the latest lbrynet-daemon release.
FROM ubuntu:18.04 as prep
LABEL MAINTAINER="leopere [at] nixc [dot] us"
RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
## Add lbrynet
ARG VERSION="latest"
RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
COPY start.sh /usr/bin/start
COPY checkmount.sh /usr/bin/checkmount
RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \
mv /lbrynet/lbrynet /usr/bin && \
chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet
FROM ubuntu:18.04 as app
COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/
RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet
## Daemon port [Intended for internal use]
## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs.
## Expose 5566 Reflector port to listen on
## Expose 5279 Port the daemon API will listen on
## the lbryumx aka Wallet port [Intended for internal use]
#EXPOSE 4444 3333 5566 5279 50001
USER lbrynet
ENTRYPOINT ["/usr/bin/checkmount"]
CMD ["start"]
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
if [ $# -eq 0 ]
then
echo "No docker tag argument supplied. Use './build.sh <tag>'"
exit 1
fi
docker build --build-arg VERSION=$1 --tag lbry/lbrynet:$1 .
docker push lbry/lbrynet:$1
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh
mountpoint=/home/lbrynet
if ! grep -qs ".* $mountpoint " /proc/mounts; then
echo "$mountpoint not mounted, refusing to run."
## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume.
exit 1
else
bash -c "$*"
fi
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
lbrynet start \
--api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \
--config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}"
+13
View File
@@ -0,0 +1,13 @@
#blockchain_name: lbrycrd_main
#blockchain_name: lbrycrd_testnet
blockchain_name: lbrycrd_regtest
lbryum_servers:
# - spv1.lbry.com:50001 #Production Wallet Server
- walletserver:50001
save_blobs: true
save_files: false
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
share_usage_data: false
tcp_port: 3333
udp_port: 4444
use_upnp: true
+12
View File
@@ -0,0 +1,12 @@
BINARY=supporty
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = ${DIR}
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} supporty.go
chmod +x ${BIN_DIR}/${BINARY}
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"os"
"strconv"
"strings"
"github.com/lbryio/ytsync/v5/util"
"github.com/sirupsen/logrus"
)
func main() {
if len(os.Args) != 6 {
logrus.Info(strings.Join(os.Args, ","))
logrus.Fatal("Not enough arguments: name, claimID, address, blockchainName, claimAmount")
}
println("Supporty!")
lbrycrd, err := util.GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
if err != nil {
logrus.Fatal(err)
}
if lbrycrd == nil {
logrus.Fatal("Lbrycrd Client is nil")
}
amount, err := strconv.ParseFloat(os.Args[5], 64)
if err != nil {
logrus.Error(err)
}
name := os.Args[1]
claimid := os.Args[2]
claimAddress := os.Args[3]
blockChainName := os.Args[4]
logrus.Infof("Supporting %s[%s] with %.2f LBC on chain %s at address %s", name, claimid, amount, blockChainName, claimAddress)
hash, err := lbrycrd.SupportClaim(name, claimid, claimAddress, blockChainName, amount)
if err != nil {
logrus.Error(err)
}
if hash == nil {
logrus.Fatal("Tx not created!")
}
logrus.Info("Tx: ", hash.String())
}
+35
View File
@@ -0,0 +1,35 @@
version: "3"
networks:
lbry-network:
external: true
services:
###################
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.73.1
restart: always
networks:
lbry-network:
ipv4_address: 10.6.1.2
volumes:
- "../persist/data/.walletserver/database:/database"
environment:
- DB_DIRECTORY=/database
- MAX_SEND=1000000000000000000000
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
- MAX_SUBS=1000000000000
- BANDWIDTH_LIMIT=80000000000
- SESSION_TIMEOUT=10000000000000000000000000
- TCP_PORT=50001
#network_mode: host
#network_mode: bridge
ports:
- "50001:50001"
expose:
- "50001"
ulimits:
nofile: 90000
# command: lbry.wallet.server.coin.LBC
command: lbry.wallet.server.coin.LBCRegTest
+39 -41
View File
@@ -1,44 +1,42 @@
module github.com/lbryio/ytsync
module github.com/lbryio/ytsync/v5
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
//replace github.com/lbryio/lbry.go/v2 => /home/niko/go/src/github.com/lbryio/lbry.go/
//replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
require (
cloud.google.com/go v0.37.4 // indirect
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/aws/aws-sdk-go v1.17.3
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/go-ini/ini v1.42.0 // indirect
github.com/golang/protobuf v1.3.1 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
github.com/gorilla/websocket v1.4.0 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.9
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/prometheus/common v0.3.0
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
github.com/sirupsen/logrus v1.4.1
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85
github.com/spf13/pflag v1.0.3 // indirect
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
go.opencensus.io v0.20.2 // indirect
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
google.golang.org/api v0.3.2
google.golang.org/appengine v1.5.0 // indirect
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
google.golang.org/grpc v1.20.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/ini.v1 v1.42.0 // indirect
gopkg.in/yaml.v2 v2.2.2 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/abadojack/whatlanggo v1.0.1
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
github.com/aws/aws-sdk-go v1.25.9
github.com/davecgh/go-spew v1.1.1
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
github.com/prometheus/client_golang v0.9.3
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0
github.com/vbauerster/mpb/v7 v7.0.2
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/ini.v1 v1.60.2 // indirect
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
gotest.tools v2.2.0+incompatible
)
go 1.13
+582 -130
View File
@@ -1,308 +1,760 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q=
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY=
github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc=
github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM=
github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc=
github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v1.0.9 h1:SAqopNiISazYuF38quA3WEfqOCzFtUed6QfCkAYQoJo=
github.com/lbryio/lbry.go v1.0.9/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b h1:/UnviTRIEwmkdLego+OI3Q75kKcztLuqPuJBYtcvz7w=
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b/go.mod h1:K7IAMJMjDisynAcDkDdZHbZeWth6sTLl66xh+uJqtGg=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262 h1:eC5o6fJT9k+RcE0aDwKyPaE4GttUC9fa7uUNNN6mzZg=
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262/go.mod h1:4zX+MQt3s9Hi1j01idC+hPWrmcokDSp8VDTIuo9Eodg=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk=
github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A=
github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU=
github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8Ma8CLF2Zzgjfx95gs3/GN3vYAPpo=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5ANXK6ozjoK47uI3uNbXv4YVINBnGm8=
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457 h1:S6VzSeZkOv8pKevvKF1isVp9HF294LxVsYw5JrV0FdQ=
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457/go.mod h1:Z/EmaCtMz3DxZsLaTnbHjqjZrOH4MHQBvvxtMObRGAY=
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4=
github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0=
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vbauerster/mpb/v7 v7.0.2 h1:eN6AD/ytv1nqCO7Dm8MO0/pGMKmMyH/WMnTJhAUuc/w=
github.com/vbauerster/mpb/v7 v7.0.2/go.mod h1:Mnq3gESXJ9eQhccbGZDggJ1faTCrmaA4iN57fUloRGE=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg=
github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ=
github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/vansante/go-ffprobe.v2 v2.0.2 h1:DdxSfFnlqeawPIVbIQEI6LR6OQHQNR7tNgWb2mWuC4w=
gopkg.in/vansante/go-ffprobe.v2 v2.0.2/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
+227
View File
@@ -0,0 +1,227 @@
package ip_manager
import (
"net"
"sort"
"sync"
"time"
"github.com/asaskevich/govalidator"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/ytsync/v5/util"
log "github.com/sirupsen/logrus"
)
const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 48 * time.Hour
var stopper = stop.New()
type IPPool struct {
ips []throttledIP
lock *sync.RWMutex
stopGrp *stop.Group
}
type throttledIP struct {
IP string
UsedForVideo string
LastUse time.Time
Throttled bool
InUse bool
}
var ipPoolInstance *IPPool
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
if ipPoolInstance != nil {
return ipPoolInstance, nil
}
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, errors.Err(err)
}
var pool []throttledIP
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
}
}
}
ipPoolInstance = &IPPool{
ips: pool,
lock: &sync.RWMutex{},
stopGrp: stopGrp,
}
//ticker := time.NewTicker(10 * time.Second)
//go func() {
// for {
// select {
// case <-stopGrp.Ch():
// return
// case <-ticker.C:
// ipPoolInstance.lock.RLock()
// for _, ip := range ipPoolInstance.ips {
// log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
// }
// ipPoolInstance.lock.RUnlock()
// }
// }
//}()
return ipPoolInstance, nil
}
// AllThrottled checks whether the IPs provided are all throttled.
// returns false if at least one IP is not throttled
// Not thread safe, should use locking when called
func AllThrottled(ips []throttledIP) bool {
for _, i := range ips {
if !i.Throttled {
return false
}
}
return true
}
// AllInUse checks whether the IPs provided are all currently in use.
// returns false if at least one IP is not in use AND is not throttled
// Not thread safe, should use locking when called
func AllInUse(ips []throttledIP) bool {
for _, i := range ips {
if !i.InUse && !i.Throttled {
return false
}
}
return true
}
func (i *IPPool) ReleaseIP(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
for j := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
localIP.InUse = false
localIP.LastUse = time.Now()
return
}
}
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
}
func (i *IPPool) ReleaseAll() {
i.lock.Lock()
defer i.lock.Unlock()
for j := range i.ips {
if i.ips[j].Throttled {
continue
}
localIP := &i.ips[j]
localIP.InUse = false
}
}
func (i *IPPool) SetThrottled(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
var tIP *throttledIP
for j, _ := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
if localIP.Throttled {
return
}
localIP.Throttled = true
tIP = localIP
break
}
}
util.SendErrorToSlack("%s set to throttled", ip)
stopper.Add(1)
go func(tIP *throttledIP) {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
i.lock.Lock()
tIP.Throttled = false
i.lock.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-i.stopGrp.Ch():
unbanTimer.Stop()
}
}(tIP)
}
var ErrAllInUse = errors.Base("all IPs are in use, try again")
var ErrAllThrottled = errors.Base("all IPs are throttled")
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
var ErrInterruptedByUser = errors.Base("interrupted by user")
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
i.lock.Lock()
defer i.lock.Unlock()
sort.Slice(i.ips, func(j, k int) bool {
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
})
if !AllThrottled(i.ips) {
if AllInUse(i.ips) {
return nil, errors.Err(ErrAllInUse)
}
var nextIP *throttledIP
for j := range i.ips {
ip := &i.ips[j]
if ip.InUse || ip.Throttled {
continue
}
nextIP = ip
break
}
if nextIP == nil {
return nil, errors.Err(ErrResourceLock)
}
nextIP.InUse = true
nextIP.UsedForVideo = forVideo
return nextIP, nil
}
return nil, errors.Err(ErrAllThrottled)
}
func (i *IPPool) GetIP(forVideo string) (string, error) {
for {
ip, err := i.nextIP(forVideo)
if err != nil {
if errors.Is(err, ErrAllInUse) {
select {
case <-i.stopGrp.Ch():
return "", errors.Err(ErrInterruptedByUser)
default:
time.Sleep(5 * time.Second)
continue
}
} else if errors.Is(err, ErrAllThrottled) {
return "throttled", err
}
return "", err
}
if time.Since(ip.LastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
}
return ip.IP, nil
}
}
+40
View File
@@ -0,0 +1,40 @@
package ip_manager
import (
"testing"
)
func TestAll(t *testing.T) {
pool, err := GetIPPool()
if err != nil {
t.Fatal(err)
}
ip, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
t.Log(ip)
pool.ReleaseIP(ip)
ip2, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
if ip == ip2 && len(pool.ips) > 1 {
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
}
t.Log(ip2)
pool.ReleaseIP(ip2)
for range pool.ips {
_, err = pool.GetIP()
if err != nil {
t.Fatal(err)
}
}
next, err := pool.nextIP()
if err != nil {
t.Logf("%s", err.Error())
} else {
t.Fatal(next)
}
}
+22
View File
@@ -0,0 +1,22 @@
package local
import (
"fmt"
"github.com/spf13/cobra"
)
func AddCommand(rootCmd *cobra.Command) {
cmd := &cobra.Command{
Use: "local",
Short: "run a personal ytsync",
Run: localCmd,
}
//cmd.Flags().StringVar(&cache, "cache", "", "path to cache")
rootCmd.AddCommand(cmd)
}
func localCmd(cmd *cobra.Command, args []string) {
fmt.Println("local")
}
+53
View File
@@ -0,0 +1,53 @@
# Running ytsync locally
## Requirements
- LBRY SDK (what do we actually need this for?)
- youtube-dl
- enough space to cache stuff
## Process
### Ensuring requirements are met
- claim channel if there isn't one yet
- or easier, just error if no channel
- enough lbc in wallet?
### Options to figure out what's already synced
- simplest: assume nothing is synced yet
- assume everything before some video is synced
- get/put sync info from Odysee by proving you have private key for channel
- tag videos as having been synced from youtube so we can ensure accuracy
- hardest: scan channel and try to match up which videos are not synced yet
### Central DB
- prove you have a channel's private key to get info about that channel
- proper queue instead of sleeping for N minutes between syncs
### Syncing a single video
- downloading it
- thumbnails
- metadata
- having enough LBC for publish(es)
- automated error handling
- getting a human involved for errors that can't be handled automatically
- reflecting
### Continuous Sync
- running in background
- storing local state
- interactions with our central ytsync db
- dealing with yt throttling
### Debugging
- dry-running the whole thing
+8 -187
View File
@@ -1,203 +1,24 @@
package main
import (
"fmt"
"math/rand"
"os"
"os/user"
"net/http"
"time"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/sdk"
"github.com/spf13/cobra"
"github.com/lbryio/ytsync/v5/cmd"
"github.com/lbryio/ytsync/manager"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
)
var Version string
const defaultMaxTries = 3
var (
stopOnError bool
maxTries int
takeOverExistingChannel bool
refill int
limit int
skipSpaceCheck bool
syncUpdate bool
singleRun bool
syncStatus string
channelID string
syncFrom int64
syncUntil int64
concurrentJobs int
videosLimit int
maxVideoSize int
maxVideoLength float64
removeDBUnpublished bool
upgradeMetadata bool
)
func main() {
rand.Seed(time.Now().UnixNano())
log.SetLevel(log.DebugLevel)
cmd := &cobra.Command{
Use: "ytsync",
Short: "Publish youtube channels into LBRY network automatically.",
Run: ytSync,
Args: cobra.RangeArgs(0, 0),
}
go func() {
http.Handle("/metrics", promhttp.Handler())
log.Error(http.ListenAndServe(":2112", nil))
}()
cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
if err := cmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func ytSync(cmd *cobra.Command, args []string) {
var hostname string
slackToken := os.Getenv("SLACK_TOKEN")
if slackToken == "" {
log.Error("A slack token was not present in env vars! Slack messages disabled!")
} else {
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync-unknown"
}
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
}
if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
return
}
if stopOnError && maxTries != defaultMaxTries {
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
return
}
if maxTries < 1 {
log.Errorln("setting --max-tries less than 1 doesn't make sense")
return
}
if limit < 0 {
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
return
}
apiURL := os.Getenv("LBRY_WEB_API")
apiToken := os.Getenv("LBRY_API_TOKEN")
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
blobsDir := os.Getenv("BLOBS_DIRECTORY")
lbrycrdString := os.Getenv("LBRYCRD_STRING")
awsS3ID := os.Getenv("AWS_S3_ID")
awsS3Secret := os.Getenv("AWS_S3_SECRET")
awsS3Region := os.Getenv("AWS_S3_REGION")
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
if apiURL == "" {
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
return
}
if apiToken == "" {
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
return
}
if youtubeAPIKey == "" {
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
return
}
if awsS3ID == "" {
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
return
}
if awsS3Secret == "" {
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
return
}
if awsS3Region == "" {
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
return
}
if awsS3Bucket == "" {
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
return
}
if lbrycrdString == "" {
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
}
if blobsDir == "" {
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return
}
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
}
syncProperties := &sdk.SyncProperties{
SyncFrom: syncFrom,
SyncUntil: syncUntil,
YoutubeChannelID: channelID,
}
apiConfig := &sdk.APIConfig{
YoutubeAPIKey: youtubeAPIKey,
ApiURL: apiURL,
ApiToken: apiToken,
HostName: hostname,
}
sm := manager.NewSyncManager(
stopOnError,
maxTries,
takeOverExistingChannel,
refill,
limit,
skipSpaceCheck,
syncUpdate,
concurrentJobs,
concurrentJobs,
blobsDir,
videosLimit,
maxVideoSize,
lbrycrdString,
awsS3ID,
awsS3Secret,
awsS3Region,
awsS3Bucket,
syncStatus,
singleRun,
syncProperties,
apiConfig,
maxVideoLength,
removeDBUnpublished,
upgradeMetadata,
)
err := sm.Start()
if err != nil {
manager.SendErrorToSlack(err.Error())
}
manager.SendInfoToSlack("Syncing process terminated!")
cmd.Execute()
}
-32
View File
@@ -1,32 +0,0 @@
package manager
import (
"net/http"
"github.com/lbryio/lbry.go/extras/errors"
"google.golang.org/api/googleapi/transport"
"google.golang.org/api/youtube/v3"
)
func (s *Sync) CountVideos() (uint64, error) {
client := &http.Client{
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
}
service, err := youtube.New(client)
if err != nil {
return 0, errors.Prefix("error creating YouTube service", err)
}
response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
if err != nil {
return 0, errors.Prefix("error getting channels", err)
}
if len(response.Items) < 1 {
return 0, errors.Err("youtube channel not found")
}
return response.Items[0].Statistics.VideoCount, nil
}
+110 -149
View File
@@ -2,188 +2,143 @@ package manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"strings"
"sync"
"syscall"
"time"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/v5/blobs_reflector"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
log "github.com/sirupsen/logrus"
)
type SyncManager struct {
stopOnError bool
maxTries int
takeOverExistingChannel bool
refill int
limit int
skipSpaceCheck bool
syncUpdate bool
concurrentJobs int
concurrentVideos int
blobsDir string
videosLimit int
maxVideoSize int
maxVideoLength float64
lbrycrdString string
awsS3ID string
awsS3Secret string
awsS3Region string
syncStatus string
awsS3Bucket string
singleRun bool
syncProperties *sdk.SyncProperties
apiConfig *sdk.APIConfig
removeDBUnpublished bool
upgradeMetadata bool
CliFlags shared.SyncFlags
ApiConfig *sdk.APIConfig
LbrycrdDsn string
AwsConfigs *shared.AwsConfigs
blobsDir string
channelsToSync []Sync
}
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
return &SyncManager{
stopOnError: stopOnError,
maxTries: maxTries,
takeOverExistingChannel: takeOverExistingChannel,
refill: refill,
limit: limit,
skipSpaceCheck: skipSpaceCheck,
syncUpdate: syncUpdate,
concurrentJobs: concurrentJobs,
concurrentVideos: concurrentVideos,
blobsDir: blobsDir,
videosLimit: videosLimit,
maxVideoSize: maxVideoSize,
maxVideoLength: maxVideoLength,
lbrycrdString: lbrycrdString,
awsS3ID: awsS3ID,
awsS3Secret: awsS3Secret,
awsS3Region: awsS3Region,
awsS3Bucket: awsS3Bucket,
syncStatus: syncStatus,
singleRun: singleRun,
syncProperties: syncProperties,
apiConfig: apiConfig,
removeDBUnpublished: removeDBUnpublished,
upgradeMetadata: upgradeMetadata,
CliFlags: cliFlags,
blobsDir: blobsDir,
LbrycrdDsn: lbrycrdDsn,
AwsConfigs: awsConfigs,
ApiConfig: apiConfig,
}
}
const (
StatusPending = "pending" // waiting for permission to sync
StatusPendingEmail = "pendingemail" // permission granted but missing email
StatusQueued = "queued" // in sync queue. will be synced soon
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
StatusSyncing = "syncing" // syncing now
StatusSynced = "synced" // done
StatusFailed = "failed"
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
)
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUnpublished = "unpublished"
)
func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
s.channelsToSync = append(s.channelsToSync, Sync{
DbChannelData: channel,
Manager: s,
namer: namer.NewNamer(),
hardVideoFailure: hardVideoFailure{
lock: &sync.Mutex{},
},
})
}
func (s *SyncManager) Start() error {
syncCount := 0
for {
err := s.checkUsedSpace()
if logUtils.ShouldCleanOnStartup() {
err := logUtils.CleanForStartup()
if err != nil {
return err
}
}
var syncs []Sync
var (
lastChannelProcessed string
secondLastChannelProcessed string
syncCount int
)
for {
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
err := s.checkUsedSpace()
if err != nil {
return errors.Err(err)
}
shouldInterruptLoop := false
isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
if isSingleChannelSync {
channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
if s.CliFlags.IsSingleChannelSync() {
channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
if err != nil {
return err
return errors.Err(err)
}
if len(channels) != 1 {
return errors.Err("Expected 1 channel, %d returned", len(channels))
}
lbryChannelName := channels[0].DesiredChannelName
syncs = make([]Sync, 1)
syncs[0] = Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
LbryChannelName: lbryChannelName,
lbryChannelID: channels[0].ChannelClaimID,
StopOnError: s.stopOnError,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
TakeOverExistingChannel: s.takeOverExistingChannel,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: channels[0].Fee,
}
s.enqueueChannel(&channels[0])
shouldInterruptLoop = true
} else {
var queuesToSync []string
if s.syncStatus != "" {
queuesToSync = append(queuesToSync, s.syncStatus)
} else if s.syncUpdate {
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
if s.CliFlags.Status != "" {
queuesToSync = append(queuesToSync, shared.StatusSyncing, s.CliFlags.Status)
} else if s.CliFlags.SyncUpdate {
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
} else {
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
}
if s.CliFlags.SecondaryStatus != "" {
queuesToSync = append(queuesToSync, s.CliFlags.SecondaryStatus)
}
queues:
for _, q := range queuesToSync {
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
if err != nil {
return err
}
log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
for _, c := range channels {
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: c.ChannelId,
LbryChannelName: c.DesiredChannelName,
lbryChannelID: c.ChannelClaimID,
StopOnError: s.stopOnError,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
TakeOverExistingChannel: s.takeOverExistingChannel,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: c.Fee,
})
s.enqueueChannel(&c)
queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
if !queueAll {
break queues
}
}
log.Infof("Drained the \"%s\" queue", q)
}
}
if len(syncs) == 0 {
if len(s.channelsToSync) == 0 {
log.Infoln("No channels to sync. Pausing 5 minutes!")
time.Sleep(5 * time.Minute)
}
for i, sync := range syncs {
for _, sync := range s.channelsToSync {
if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
stopTheLoops := errors.Err("Found channel %s running 3 times, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
sync.setChannelTerminationStatus(&stopTheLoops)
continue
}
secondLastChannelProcessed = lastChannelProcessed
lastChannelProcessed = sync.DbChannelData.ChannelId
shouldNotCount := false
SendInfoToSlack("Syncing %s (%s) to LBRY! (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
err := sync.FullCycle()
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
ipPool, _ := ip_manager.GetIPPool(sync.grp)
if ipPool != nil {
ipPool.ReleaseAll()
}
if err != nil {
if strings.Contains(err.Error(), "quotaExceeded") {
logUtils.SleepUntilQuotaReset()
}
fatalErrors := []string{
"default_wallet already exists",
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
@@ -192,42 +147,48 @@ func (s *SyncManager) Start() error {
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
}
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
if !shouldNotCount {
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
}
}
SendInfoToSlack("Syncing %s (%s) reached an end. (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
err = logUtils.CleanupMetadata()
if err != nil {
log.Errorf("something went wrong while trying to clear out the video metadata directory: %s", errors.FullTrace(err))
}
err = blobs_reflector.ReflectAndClean()
if err != nil {
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
}
logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
if !shouldNotCount {
syncCount++
}
if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
shouldInterruptLoop = true
break
}
}
if shouldInterruptLoop || s.singleRun {
if shouldInterruptLoop || s.CliFlags.SingleRun {
break
}
}
return nil
}
func (s *SyncManager) GetS3AWSConfig() aws.Config {
return aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
Region: &s.awsS3Region,
}
}
func (s *SyncManager) checkUsedSpace() error {
usedPctile, err := GetUsedSpace(s.blobsDir)
usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
if err != nil {
return err
return errors.Err(err)
}
if usedPctile >= 0.90 && !s.skipSpaceCheck {
if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
}
log.Infof("disk usage: %.1f%%", usedPctile*100)
+254
View File
@@ -0,0 +1,254 @@
package manager
import (
"os"
"path/filepath"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
logUtils "github.com/lbryio/ytsync/v5/util"
)
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return nil, errors.Prefix("error starting session: ", err)
}
downloader := s3manager.NewDownloader(s3Session)
return downloader, nil
}
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return nil, errors.Prefix("error starting session: ", err)
}
uploader := s3manager.NewUploader(s3Session)
return uploader, nil
}
func (s *Sync) downloadWallet() error {
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
if err != nil {
return errors.Err(err)
}
downloader, err := s.getS3Downloader()
if err != nil {
return err
}
out, err := os.Create(defaultTempWalletDir)
if err != nil {
return errors.Prefix("error creating temp wallet: ", err)
}
defer out.Close()
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
})
if err != nil {
// Casting to the awserr.Error type will allow you to inspect the error
// code returned by the service in code. The error code can be used
// to switch on context specific functionality. In this case a context
// specific error message is printed to the user based on the bucket
// and key existing.
//
// For information on other S3 API error codes see:
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
if aerr, ok := err.(awserr.Error); ok {
code := aerr.Code()
if code == s3.ErrCodeNoSuchKey {
return errors.Err("wallet not on S3")
}
}
return err
} else if bytesWritten == 0 {
return errors.Err("zero bytes written")
}
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
if err != nil {
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
}
return nil
}
func (s *Sync) downloadBlockchainDB() error {
if logUtils.IsRegTest() {
return nil // tests fail if we re-use the same blockchain DB
}
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
if err != nil {
return errors.Err(err)
}
files, err := filepath.Glob(defaultBDBDir + "*")
if err != nil {
return errors.Err(err)
}
for _, f := range files {
err = os.Remove(f)
if err != nil {
return errors.Err(err)
}
}
if s.DbChannelData.WipeDB {
return nil
}
downloader, err := s.getS3Downloader()
if err != nil {
return errors.Err(err)
}
out, err := os.Create(defaultTempBDBDir)
if err != nil {
return errors.Prefix("error creating temp wallet: ", err)
}
defer out.Close()
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
})
if err != nil {
// Casting to the awserr.Error type will allow you to inspect the error
// code returned by the service in code. The error code can be used
// to switch on context specific functionality. In this case a context
// specific error message is printed to the user based on the bucket
// and key existing.
//
// For information on other S3 API error codes see:
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
if aerr, ok := err.(awserr.Error); ok {
code := aerr.Code()
if code == s3.ErrCodeNoSuchKey {
return nil // let ytsync sync the database by itself
}
}
return errors.Err(err)
} else if bytesWritten == 0 {
return errors.Err("zero bytes written")
}
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
if err != nil {
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
}
log.Printf("blockchain.db downloaded to %s", defaultBDBDir)
return nil
}
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
}
lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir != "" {
defaultWallet = lbryumDir + "/wallets/default_wallet"
tempWallet = lbryumDir + "/wallets/tmp_wallet"
}
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
return "", "", nil, errors.Err("default_wallet already exists")
}
return
}
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir == "" {
if logUtils.IsRegTest() {
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
} else {
lbryumDir = os.Getenv("HOME") + "/.lbryum"
}
}
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
}
return
}
func (s *Sync) uploadWallet() error {
defaultWalletDir := logUtils.GetDefaultWalletPath()
key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
}
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
return errors.Err("default_wallet does not exist")
}
uploader, err := s.getS3Uploader()
if err != nil {
return err
}
file, err := os.Open(defaultWalletDir)
if err != nil {
return err
}
defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
Body: file,
})
if err != nil {
return err
}
log.Println("wallet uploaded to S3")
return os.Remove(defaultWalletDir)
}
func (s *Sync) uploadBlockchainDB() error {
defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
if err != nil {
return errors.Err(err)
}
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
return errors.Err("blockchain.db does not exist")
}
uploader, err := s.getS3Uploader()
if err != nil {
return err
}
file, err := os.Open(defaultBDBDir)
if err != nil {
return err
}
defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
Body: file,
})
if err != nil {
return err
}
log.Println("blockchain.db uploaded to S3")
return os.Remove(defaultBDBDir)
}
+269 -200
View File
@@ -2,39 +2,46 @@ package manager
import (
"fmt"
"net/http"
"os"
"math"
"strconv"
"strings"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/lbry.go/lbrycrd"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/ytsync/v5/ytapi"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/googleapi/transport"
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 3
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
return errors.Err(err)
}
accounts := accountsResponse.LBCMainnet
if os.Getenv("REGTEST") == "true" {
accounts = accountsResponse.LBCRegtest
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
ledger := "lbc_mainnet"
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
accounts = append(accounts, a)
}
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: 1000,
ChangeMaxUses: util.PtrToInt(1000),
ReceivingMaxUses: util.PtrToInt(100),
})
if err != nil {
return errors.Err(err)
@@ -43,7 +50,11 @@ func (s *Sync) enableAddressReuse() error {
return nil
}
func (s *Sync) walletSetup() error {
//prevent unnecessary concurrent execution
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("walletSetup").Add(time.Since(start))
}(start)
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
s.walletMux.Lock()
defer s.walletMux.Unlock()
err := s.ensureChannelOwnership()
@@ -57,74 +68,78 @@ func (s *Sync) walletSetup() error {
} else if balanceResp == nil {
return errors.Err("no response")
}
balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
balance, err := strconv.ParseFloat(balanceResp.Available.String(), 64)
if err != nil {
return errors.Err(err)
}
log.Debugf("Starting balance is %.4f", balance)
n, err := s.CountVideos()
if err != nil {
return err
}
numOnSource := int(n)
videosOnYoutube := int(s.DbChannelData.TotalVideos)
log.Debugf("Source channel has %d videos", numOnSource)
if numOnSource == 0 {
log.Debugf("Source channel has %d videos", videosOnYoutube)
if videosOnYoutube == 0 {
return nil
}
s.syncedVideosMux.RLock()
numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones...
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d videos", numPublished)
if numOnSource-numPublished > s.Manager.videosLimit {
numOnSource = s.Manager.videosLimit
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if s.Manager.upgradeMetadata {
videosToUpgrade := 0
for _, v := range s.syncedVideos {
if v.Published && v.MetadataVersion < 2 {
videosToUpgrade++
publishedCount := 0
notUpgradedCount := 0
failedCount := 0
for _, sv := range s.syncedVideos {
if sv.Published {
publishedCount++
if sv.MetadataVersion < 2 {
notUpgradedCount++
}
}
minBalance += float64(videosToUpgrade) * 0.001
}
if numPublished > numOnSource && balance < minimumRefillAmount {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = minimumRefillAmount
}
amountToAdd := minBalance - balance
if s.Refill > 0 {
if amountToAdd < 0 {
amountToAdd = float64(s.Refill)
} else {
amountToAdd += float64(s.Refill)
failedCount++
}
}
s.syncedVideosMux.RUnlock()
if amountToAdd > 0 {
if amountToAdd < minimumRefillAmount {
amountToAdd = minimumRefillAmount
}
err := s.addCredits(amountToAdd)
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
if videosOnYoutube > s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers) {
videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
channelFee := channelClaimAmount
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
if channelAlreadyClaimed {
channelFee = 0.0
}
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
if s.Manager.CliFlags.UpgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
}
refillAmount := 0.0
if balance < requiredBalance || balance < minimumAccountBalance {
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
}
if s.Manager.CliFlags.Refill > 0 {
refillAmount += float64(s.Manager.CliFlags.Refill)
}
if refillAmount > 0 {
err := s.addCredits(refillAmount)
if err != nil {
return errors.Err(err)
}
}
claimAddress, err := s.daemon.AddressList(nil)
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
if err != nil {
return err
} else if claimAddress == nil {
return errors.Err("could not get unused address")
return errors.Err("could not get an address")
}
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
if s.claimAddress == "" {
if s.DbChannelData.PublishAddress.Address == "" || !s.shouldTransfer() {
s.DbChannelData.PublishAddress.Address = string(claimAddress.Items[0].Address)
s.DbChannelData.PublishAddress.IsMine = true
}
if s.DbChannelData.PublishAddress.Address == "" {
return errors.Err("found blank claim address")
}
@@ -136,27 +151,47 @@ func (s *Sync) walletSetup() error {
return nil
}
func (s *Sync) ensureEnoughUTXOs() error {
accounts, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accountsNet := (*accounts).LBCMainnet
if os.Getenv("REGTEST") == "true" {
accountsNet = (*accounts).LBCRegtest
}
defaultAccount := ""
for _, account := range accountsNet {
if account.IsDefault {
defaultAccount = account.ID
break
func (s *Sync) getDefaultAccount() (string, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
}(start)
if s.defaultAccountID == "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
return "", errors.Err(err)
}
ledger := "lbc_mainnet"
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.IsDefault {
s.defaultAccountID = a.ID
break
}
}
}
if s.defaultAccountID == "" {
return "", errors.Err("No default account found")
}
}
if defaultAccount == "" {
return errors.Err("No default account found")
return s.defaultAccountID, nil
}
func (s *Sync) ensureEnoughUTXOs() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
}(start)
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
utxolist, err := s.daemon.UTXOList(&defaultAccount)
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
if err != nil {
return err
} else if utxolist == nil {
@@ -166,14 +201,19 @@ func (s *Sync) ensureEnoughUTXOs() error {
target := 40
slack := int(float32(0.1) * float32(target))
count := 0
confirmedCount := 0
for _, utxo := range *utxolist {
for _, utxo := range utxolist.Items {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
confirmedCount++
}
count++
}
}
log.Infof("utxo count: %d", count)
log.Infof("utxo count: %d (%d confirmed)", count, confirmedCount)
UTXOWaitThreshold := 16
if count < target-slack {
balance, err := s.daemon.AccountBalance(&defaultAccount)
if err != nil {
@@ -182,27 +222,44 @@ func (s *Sync) ensureEnoughUTXOs() error {
return errors.Err("no response")
}
balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64)
if err != nil {
return errors.Err(err)
}
broadcastFee := 0.01
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
//this is dumb but sometimes the balance is negative and it breaks everything, so let's check again
if balanceAmount < 0 {
log.Infof("negative balance of %.2f found. Waiting to retry...", balanceAmount)
time.Sleep(10 * time.Second)
balanceAmount, err = strconv.ParseFloat(balance.Available.String(), 64)
if err != nil {
return errors.Err(err)
}
}
maxUTXOs := uint64(500)
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
if desiredUTXOCount > maxUTXOs {
desiredUTXOCount = maxUTXOs
}
if desiredUTXOCount < uint64(confirmedCount) {
return nil
}
availableBalance, _ := balance.Available.Float64()
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
broadcastFee := 0.1
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
if err != nil {
return err
} else if prefillTx == nil {
return errors.Err("no response")
}
err = s.waitForNewBlock()
if err != nil {
return err
if confirmedCount < UTXOWaitThreshold {
err = s.waitForNewBlock()
if err != nil {
return err
}
}
} else if !allUTXOsConfirmed(utxolist) {
} else if confirmedCount < UTXOWaitThreshold {
log.Println("Waiting for previous txns to confirm")
err := s.waitForNewBlock()
if err != nil {
@@ -214,6 +271,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
func (s *Sync) waitForNewBlock() error {
defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
status, err := s.daemon.Status()
if err != nil {
return err
@@ -226,68 +286,89 @@ func (s *Sync) waitForNewBlock() error {
return err
}
}
currentBlock := status.Wallet.Blocks
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
if i%3 == 0 {
log.Printf("Waiting for new block (%d)...", currentBlock+1)
}
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
err = s.GenerateRegtestBlock()
if err != nil {
return err
}
}
time.Sleep(10 * time.Second)
status, err = s.daemon.Status()
if err != nil {
return err
}
}
time.Sleep(5 * time.Second)
return nil
}
func (s *Sync) GenerateRegtestBlock() error {
lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block: ", err)
}
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
return nil
}
func (s *Sync) ensureChannelOwnership() error {
if s.LbryChannelName == "" {
defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
if s.DbChannelData.DesiredChannelName == "" {
return errors.Err("no channel name set")
}
//@TODO: get rid of this when imported channels are supported
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
return nil
}
channels, err := s.daemon.ChannelList(nil, 1, 50)
channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
if err != nil {
return err
} else if channels == nil {
return errors.Err("no channel response")
}
//special case for wallets we don't retain full control anymore
if len((*channels).Items) > 1 {
// This wallet is probably not under our control anymore but we still want to publish to it
// here we shall check if within all the channels there is one that was created by ytsync
SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
if s.lbryChannelID == "" {
var channelToUse *jsonrpc.Transaction
if len((*channels).Items) > 0 {
if s.DbChannelData.ChannelClaimID == "" {
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
}
for _, c := range (*channels).Items {
if c.ClaimID != s.lbryChannelID {
if c.Name != s.LbryChannelName {
return errors.Err("the channel in the wallet is different than the channel in the database")
}
return nil // we have the ytsync channel and both the claimID and the channelName from the database are correct
log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
if c.ClaimID != s.DbChannelData.ChannelClaimID {
continue
}
}
}
channelUsesOldMetadata := false
if len((*channels).Items) == 1 {
channel := ((*channels).Items)[0]
if channel.Name == s.LbryChannelName {
channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
//TODO: eventually get rid of this when the whole db is filled
if s.lbryChannelID == "" {
err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
} else if channel.ClaimID != s.lbryChannelID {
if c.Name != s.DbChannelData.DesiredChannelName {
return errors.Err("the channel in the wallet is different than the channel in the database")
}
s.lbryChannelID = channel.ClaimID
if !channelUsesOldMetadata {
return err
}
} else {
return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
channelToUse = &c
break
}
if channelToUse == nil {
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
}
} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
return errors.Err("the channel was transferred but appears to have been abandoned!")
} else if s.DbChannelData.ChannelClaimID != "" {
return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
}
channelUsesOldMetadata := false
if channelToUse != nil {
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
if !channelUsesOldMetadata {
return nil
}
}
@@ -299,47 +380,44 @@ func (s *Sync) ensureChannelOwnership() error {
} else if balanceResp == nil {
return errors.Err("no response")
}
balance, err := decimal.NewFromString((string)(*balanceResp))
balance, err := decimal.NewFromString(balanceResp.Available.String())
if err != nil {
return errors.Err(err)
}
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
err = s.addCredits(channelBidAmount + 0.1)
err = s.addCredits(channelBidAmount + 0.3)
if err != nil {
return err
}
}
client := &http.Client{
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
}
service, err := youtube.New(client)
channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
if err != nil {
return errors.Prefix("error creating YouTube service", err)
if strings.Contains(err.Error(), "invalid character 'e' looking for beginning of value") {
logUtils.SendInfoToSlack("failed to get channel data for %s. Waiting 1 minute to retry", s.DbChannelData.ChannelId)
time.Sleep(1 * time.Minute)
channelInfo, err = ytapi.ChannelInfo(s.DbChannelData.ChannelId)
if err != nil {
return err
}
} else {
return err
}
}
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
if err != nil {
return errors.Prefix("error getting channel details", err)
}
if len(response.Items) < 1 {
return errors.Err("youtube channel not found")
}
channelInfo := response.Items[0].Snippet
channelBranding := response.Items[0].BrandingSettings
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return err
}
var bannerURL *string
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
"banner-"+s.DbChannelData.ChannelId,
*s.Manager.AwsConfigs.GetS3AWSConfig(),
)
if err != nil {
return err
}
@@ -347,34 +425,43 @@ func (s *Sync) ensureChannelOwnership() error {
}
var languages []string = nil
if channelInfo.DefaultLanguage != "" {
languages = []string{channelInfo.DefaultLanguage}
}
//we don't have this data without the API
//if channelInfo.DefaultLanguage != "" {
// if channelInfo.DefaultLanguage == "iw" {
// channelInfo.DefaultLanguage = "he"
// }
// languages = []string{channelInfo.DefaultLanguage}
//}
var locations []jsonrpc.Location = nil
if channelInfo.Country != "" {
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
}
var c *jsonrpc.TransactionSummary
claimCreateOptions := jsonrpc.ClaimCreateOptions{
Title: &channelInfo.Title,
Description: &channelInfo.Description,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description,
Tags: tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
}
if channelUsesOldMetadata {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
},
})
if s.DbChannelData.TransferState <= 1 {
c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
},
})
} else {
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
return nil
}
} else {
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
})
@@ -383,45 +470,27 @@ func (s *Sync) ensureChannelOwnership() error {
if err != nil {
return err
}
s.lbryChannelID = c.Outputs[0].ClaimID
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
}
func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
if utxolist == nil {
return false
}
if len(*utxolist) < 1 {
return false
}
for _, utxo := range *utxolist {
if utxo.Height == 0 {
return false
}
}
return true
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
}
func (s *Sync) addCredits(amountToAdd float64) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("addCredits").Add(time.Since(start))
}(start)
log.Printf("Adding %f credits", amountToAdd)
var lbrycrdd *lbrycrd.Client
var err error
if s.LbrycrdString == "" {
lbrycrdd, err = lbrycrd.NewWithDefaultURL()
if err != nil {
return err
}
} else {
lbrycrdd, err = lbrycrd.New(s.LbrycrdString)
if err != nil {
return err
}
lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
if err != nil {
return err
}
addressResp, err := s.daemon.AddressUnused(nil)
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
if err != nil {
return err
} else if addressResp == nil {
+349
View File
@@ -0,0 +1,349 @@
package manager
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/timing"
log "github.com/sirupsen/logrus"
)
func waitConfirmations(s *Sync) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("waitConfirmations").Add(time.Since(start))
}(start)
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
allConfirmed := false
waitCount := 0
waiting:
for !allConfirmed && waitCount < 2 {
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
if err != nil {
return err
} else if utxolist == nil {
return errors.Err("no response")
}
for _, utxo := range utxolist.Items {
if utxo.Confirmations <= 0 {
err = s.waitForNewBlock()
if err != nil {
return err
}
waitCount++
continue waiting
}
}
allConfirmed = true
}
return nil
}
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
func abandonSupports(s *Sync) (float64, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("abandonSupports").Add(time.Since(start))
}(start)
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return 0, err
}
for page := uint64(1); page <= totalPages; page++ {
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
if err != nil {
supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
if err != nil {
return 0, errors.Prefix("cannot list supports", err)
}
}
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
}
producerWG := &stop.Group{}
claimIDChan := make(chan string, len(allSupports))
abandonRspChan := make(chan abandonResponse, len(allSupports))
alreadyAbandoned := make(map[string]bool, len(allSupports))
producerWG.Add(1)
go func() {
defer producerWG.Done()
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
continue
}
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
}
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(60 * time.Second)
defer s.daemon.SetRPCTimeout(5 * time.Minute)
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
consumerWG.Add(1)
go func() {
defer consumerWG.Done()
outer:
for {
claimID, more := <-claimIDChan
if !more {
return
} else {
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
} else {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0,
}
continue
}
}
if summary == nil || len(*summary) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
Amount: 0,
}
continue
}
var outputAmount float64
for _, tx := range *summary {
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue outer
}
outputAmount += amount
}
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: nil,
Amount: outputAmount,
}
continue
}
}
}()
}
producerWG.Wait()
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
totalAbandoned := 0.0
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
continue
}
totalAbandoned += r.Amount
}
return totalAbandoned, nil
}
type updateInfo struct {
ClaimID string
streamUpdateOptions *jsonrpc.StreamUpdateOptions
videoStatus *shared.VideoStatus
}
func transferVideos(s *Sync) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("transferVideos").Add(time.Since(start))
}(start)
cleanTransfer := true
streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
account, err := s.getDefaultAccount()
if err != nil {
return err
}
streams, err := s.daemon.StreamList(&account, 1, 30000)
if err != nil {
return errors.Err(err)
}
producerWG := &stop.Group{}
producerWG.Add(1)
go func() {
defer producerWG.Done()
for _, video := range s.syncedVideos {
if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
continue
}
var stream *jsonrpc.Claim = nil
for _, c := range streams.Items {
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
continue
}
stream = &c
break
}
if stream == nil {
return
}
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
ClaimAddress: &s.DbChannelData.PublishAddress.Address,
FundingAccountIDs: []string{
account,
},
},
},
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
videoStatus := shared.VideoStatus{
ChannelID: s.DbChannelData.ChannelId,
VideoID: video.VideoID,
ClaimID: video.ClaimID,
ClaimName: video.ClaimName,
Status: shared.VideoStatusPublished,
IsTransferred: util.PtrToBool(true),
}
streamChan <- updateInfo{
ClaimID: video.ClaimID,
streamUpdateOptions: &streamUpdateOptions,
videoStatus: &videoStatus,
}
}
}()
consumerWG := &stop.Group{}
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
consumerWG.Add(1)
go func(worker int) {
defer consumerWG.Done()
for {
ui, more := <-streamChan
if !more {
return
} else {
err := s.streamUpdate(&ui)
if err != nil {
cleanTransfer = false
}
}
}
}(i)
}
producerWG.Wait()
close(streamChan)
consumerWG.Wait()
if !cleanTransfer {
return errors.Err("A video has failed to transfer for the channel...skipping channel transfer")
}
return nil
}
func (s *Sync) streamUpdate(ui *updateInfo) error {
start := time.Now()
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
if updateError != nil {
ui.videoStatus.FailureReason = updateError.Error()
ui.videoStatus.Status = shared.VideoStatusTranferFailed
ui.videoStatus.IsTransferred = util.PtrToBool(false)
} else {
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
}
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
if statusErr != nil {
return errors.Prefix(statusErr.Error(), updateError)
}
return errors.Err(updateError)
}
func transferChannel(s *Sync) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("transferChannel").Add(time.Since(start))
}(start)
account, err := s.getDefaultAccount()
if err != nil {
return err
}
channelClaims, err := s.daemon.ChannelList(&account, 1, 50, nil)
if err != nil {
return errors.Err(err)
}
var channelClaim *jsonrpc.Transaction = nil
for _, c := range channelClaims.Items {
if c.ClaimID != s.DbChannelData.ChannelClaimID {
continue
}
channelClaim = &c
break
}
if channelClaim == nil {
return nil
}
updateOptions := jsonrpc.ChannelUpdateOptions{
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
ClaimAddress: &s.DbChannelData.PublishAddress.Address,
},
},
}
result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
if err != nil {
return errors.Err(err)
}
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
return nil
}
+617 -502
View File
File diff suppressed because it is too large. Load diff
+31
View File
@@ -0,0 +1,31 @@
package metrics
import (
"os"
"regexp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
)
var (
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "ytsync",
Subsystem: getHostname(),
Name: "duration",
Help: "The durations of the individual modules",
}, []string{"path"})
)
func getHostname() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "ytsync_unknown"
}
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err != nil {
log.Fatal(err)
}
return reg.ReplaceAllString(hostname, "_")
}
+30 -5
View File
@@ -10,7 +10,7 @@ import (
"sync"
)
var titleRegexp = regexp.MustCompile(`[^a-zA-Z0-9]+`)
var claimNameRegexp = regexp.MustCompile(`[=&#:$@%??;、\\"/<>%{}||^~\x60[\]\s]`)
type Namer struct {
mu *sync.Mutex
@@ -43,9 +43,16 @@ func (n *Namer) GetNextName(prefix string) string {
}
//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
attempt = 1
if len(name) < 2 {
sum := md5.Sum([]byte(prefix))
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
for {
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
if _, exists := n.names[name]; !exists {
break
}
attempt++
}
}
n.names[name] = true
@@ -61,18 +68,21 @@ func getClaimNameFromTitle(title string, attempt int) string {
}
maxLen := 40 - len(suffix)
chunks := strings.Split(strings.ToLower(strings.Trim(titleRegexp.ReplaceAllString(title, "-"), "-")), "-")
chunks := strings.Split(strings.ToLower(strings.Trim(claimNameRegexp.ReplaceAllString(title, "-"), "-")), "-")
name := chunks[0]
if len(name) > maxLen {
return name[:maxLen]
return truncateUnicode(name, maxLen) + suffix
}
for _, chunk := range chunks[1:] {
if chunk == "" {
continue
}
tmpName := name + "-" + chunk
if len(tmpName) > maxLen {
if len(name) < 20 {
name = tmpName[:maxLen]
name = truncateUnicode(tmpName, maxLen-len(name))
}
break
}
@@ -81,3 +91,18 @@ func getClaimNameFromTitle(title string, attempt int) string {
return name + suffix
}
func truncateUnicode(name string, limit int) string {
reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`)
name = reNameBlacklist.ReplaceAllString(name, "")
result := name
chars := 0
for i := range name {
if chars >= limit {
result = name[:i]
break
}
chars++
}
return result
}
+28
View File
@@ -0,0 +1,28 @@
package namer
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_getClaimNameFromTitle(t *testing.T) {
name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
assert.Equal(t, "стопхам-в-ожидании", name)
name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name)
name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG?|有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
assert.Equal(t, "錢包整理術-5-tips-哪種錢包最ng", name)
name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0)
assert.Equal(t, "اسرع-طريقة-لتختيم", name)
name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0)
assert.Equal(t, "شكرا-على-380-مشترك😍😍😍", name)
name = getClaimNameFromTitle("test-@", 0)
assert.Equal(t, "test", name)
name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0)
assert.Equal(t, "『あなたはただの空の殻でした』", name)
name = getClaimNameFromTitle("精靈樂章-這樣的夥伴沒問題嗎 幽暗隕石坑(夢魘) 王有無敵狀態...要會閃不然會被秒(無課)", 2)
assert.Equal(t, "精靈樂章-這樣的夥伴沒問題嗎-2", name)
name = getClaimNameFromTitle("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 50)
assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-50", name)
}
+267 -59
View File
@@ -11,8 +11,11 @@ import (
"strings"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/null"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/null"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/util"
log "github.com/sirupsen/logrus"
)
@@ -28,47 +31,42 @@ type APIConfig struct {
HostName string
}
type SyncProperties struct {
SyncFrom int64
SyncUntil int64
YoutubeChannelID string
}
type Fee struct {
Amount string `json:"amount"`
Address string `json:"address"`
Currency string `json:"currency"`
}
type YoutubeChannel struct {
ChannelId string `json:"channel_id"`
TotalVideos uint `json:"total_videos"`
DesiredChannelName string `json:"desired_channel_name"`
Fee *Fee `json:"fee"`
ChannelClaimID string `json:"channel_claim_id"`
}
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
type apiJobsResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data []YoutubeChannel `json:"data"`
Success bool `json:"success"`
Error null.String `json:"error"`
Data []shared.YoutubeChannel `json:"data"`
}
endpoint := a.ApiURL + "/yt/jobs"
res, _ := http.PostForm(endpoint, url.Values{
res, err := http.PostForm(endpoint, url.Values{
"auth_token": {a.ApiToken},
"sync_status": {status},
"min_videos": {strconv.Itoa(1)},
"after": {strconv.Itoa(int(cp.SyncFrom))},
"before": {strconv.Itoa(int(cp.SyncUntil))},
"after": {strconv.Itoa(int(cliFlags.SyncFrom))},
"before": {strconv.Itoa(int(cliFlags.SyncUntil))},
"sync_server": {a.HostName},
"channel_id": {cp.YoutubeChannelID},
"channel_id": {cliFlags.ChannelID},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cliFlags)
}
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cliFlags)
}
var response apiJobsResponse
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return nil, err
return nil, errors.Err(err)
}
if response.Data == nil {
return nil, errors.Err(response.Error)
@@ -85,6 +83,8 @@ type SyncedVideo struct {
ClaimID string `json:"claim_id"`
Size int64 `json:"size"`
MetadataVersion int8 `json:"metadata_version"`
Transferred bool `json:"transferred"`
IsLbryFirst bool `json:"is_lbry_first"`
}
func sanitizeFailureReason(s *string) {
@@ -95,7 +95,50 @@ func sanitizeFailureReason(s *string) {
*s = (*s)[:MaxReasonLength]
}
}
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) {
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
type apiSetChannelCertResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data string `json:"data"`
}
endpoint := a.ApiURL + "/yt/channel_cert"
res, err := http.PostForm(endpoint, url.Values{
"channel_claim_id": {channelID},
"channel_cert": {certHex},
"auth_token": {a.ApiToken},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
var response apiSetChannelCertResponse
err = json.Unmarshal(body, &response)
if err != nil {
return errors.Err(err)
}
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
return nil
}
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) {
type apiChannelStatusResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
@@ -104,19 +147,37 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
endpoint := a.ApiURL + "/yt/channel_status"
sanitizeFailureReason(&failureReason)
res, _ := http.PostForm(endpoint, url.Values{
params := url.Values{
"channel_id": {channelID},
"sync_server": {a.HostName},
"auth_token": {a.ApiToken},
"sync_status": {status},
"failure_reason": {failureReason},
})
}
if transferState != nil {
params.Add("transfer_state", strconv.Itoa(*transferState))
}
res, err := http.PostForm(endpoint, params)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
return nil, nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode >= http.StatusInternalServerError {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
var response apiChannelStatusResponse
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return nil, nil, err
return nil, nil, errors.Err(err)
}
if !response.Error.IsNull() {
return nil, nil, errors.Err(response.Error.String)
@@ -142,15 +203,29 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
Data string `json:"data"`
}
endpoint := a.ApiURL + "/yt/set_channel_claim_id"
res, _ := http.PostForm(endpoint, url.Values{
res, err := http.PostForm(endpoint, url.Values{
"channel_id": {channelID},
"auth_token": {a.ApiToken},
"channel_claim_id": {channelClaimID},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
var response apiChannelStatusResponse
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return errors.Err(err)
}
@@ -164,8 +239,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusPublished = "published"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusFailed = "failed"
)
func (a *APIConfig) DeleteVideos(videos []string) error {
@@ -175,15 +251,29 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
"video_ids": {videoIDs},
"auth_token": {a.ApiToken},
}
res, _ := http.PostForm(endpoint, vals)
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data null.String `json:"data"`
}
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return errors.Err(err)
}
@@ -197,40 +287,59 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
endpoint := a.ApiURL + "/yt/video_status"
sanitizeFailureReason(&failureReason)
sanitizeFailureReason(&status.FailureReason)
vals := url.Values{
"youtube_channel_id": {channelID},
"video_id": {videoID},
"status": {status},
"youtube_channel_id": {status.ChannelID},
"video_id": {status.VideoID},
"status": {status.Status},
"auth_token": {a.ApiToken},
}
if status == VideoStatusPublished {
if claimID == "" || claimName == "" {
return errors.Err("claimID or claimName missing")
if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed {
if status.ClaimID == "" || status.ClaimName == "" {
return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName)
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
vals.Add("claim_id", status.ClaimID)
vals.Add("claim_name", status.ClaimName)
if status.MetaDataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion))
}
if status.Size != nil {
vals.Add("size", strconv.FormatInt(*status.Size, 10))
}
}
if failureReason != "" {
vals.Add("failure_reason", failureReason)
if status.FailureReason != "" {
vals.Add("failure_reason", status.FailureReason)
}
if status.IsTransferred != nil {
vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
return errors.Err(err)
}
res, _ := http.PostForm(endpoint, vals)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data null.String `json:"data"`
}
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return err
}
@@ -242,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
}
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
func (a *APIConfig) VideoState(videoID string) (string, error) {
endpoint := a.ApiURL + "/yt/video_state"
vals := url.Values{
"video_id": {videoID},
"auth_token": {a.ApiToken},
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.VideoState(videoID)
}
return "", errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode == http.StatusNotFound {
return "not_found", nil
}
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.VideoState(videoID)
}
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data null.String `json:"data"`
}
err = json.Unmarshal(body, &response)
if err != nil {
return "", errors.Err(err)
}
if !response.Error.IsNull() {
return "", errors.Err(response.Error.String)
}
if !response.Data.IsNull() {
return response.Data.String, nil
}
return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
type VideoRelease struct {
ID uint64 `json:"id"`
YoutubeDataID uint64 `json:"youtube_data_id"`
VideoID string `json:"video_id"`
ReleaseTime string `json:"release_time"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
endpoint := a.ApiURL + "/yt/released"
vals := url.Values{
"video_id": {videoID},
"auth_token": {a.ApiToken},
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.GetReleasedDate(videoID)
}
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode == http.StatusNotFound {
return nil, nil
}
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.GetReleasedDate(videoID)
}
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data VideoRelease `json:"data"`
}
err = json.Unmarshal(body, &response)
if err != nil {
return nil, errors.Err(err)
}
if !response.Error.IsNull() {
return nil, errors.Err(response.Error.String)
}
if response.Data.ReleaseTime != "" {
return &response.Data, nil
}
return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
+229
View File
@@ -0,0 +1,229 @@
package shared
import (
"encoding/json"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
type Fee struct {
Amount string `json:"amount"`
Address string `json:"address"`
Currency string `json:"currency"`
}
type YoutubeChannel struct {
ChannelId string `json:"channel_id"`
TotalVideos uint `json:"total_videos"`
TotalSubscribers uint `json:"total_subscribers"`
DesiredChannelName string `json:"desired_channel_name"`
Fee *Fee `json:"fee"`
ChannelClaimID string `json:"channel_claim_id"`
TransferState int `json:"transfer_state"`
PublishAddress PublishAddress `json:"publish_address"`
PublicKey string `json:"public_key"`
LengthLimit int `json:"length_limit"`
SizeLimit int `json:"size_limit"`
LastUploadedVideo string `json:"last_uploaded_video"`
WipeDB bool `json:"wipe_db"`
}
type PublishAddress struct {
Address string `json:"address"`
IsMine bool `json:"is_mine"`
}
func (p *PublishAddress) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
return errors.Err(err)
}
p.Address = s
p.IsMine = false
return nil
}
var FatalErrors = []string{
":5279: read: connection reset by peer",
"no space left on device",
"NotEnoughFunds",
"Cannot publish using channel",
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"Couldn't find private key for id",
"You already have a stream claim published under the name",
"Missing inputs",
}
var ErrorsNoRetry = []string{
"Requested format is not available",
"non 200 status code received",
"This video contains content from",
"dont know which claim to update",
"uploader has not made this video available in your country",
"download error: AccessDenied: Access Denied",
"Playback on other websites has been disabled by the video owner",
"Error in daemon: Cannot publish empty file",
"Error extracting sts from embedded url response",
"Unable to extract signature tokens",
"Client.Timeout exceeded while awaiting headers",
"the video is too big to sync, skipping for now",
"video is too long to process",
"video is too short to process",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"HTTP Error 403",
"giving up after 0 fragment retries",
"Sorry about that",
"This video is not available",
"requested format not available",
"interrupted by user",
"Sign in to confirm your age",
"This video is unavailable",
"video is a live stream and hasn't completed yet",
"Premieres in",
"Private video",
"This live event will begin in",
"This video has been removed by the uploader",
"Premiere will begin shortly",
"cannot unmarshal number 0.0",
"default youtube thumbnail found",
}
var WalletErrors = []string{
"Not enough funds to cover this transaction",
"failed: Not enough funds",
"Error in daemon: Insufficient funds, please deposit additional LBC",
//"Missing inputs",
}
var BlockchainErrors = []string{
"txn-mempool-conflict",
"too-long-mempool-chain",
}
var NeverRetryFailures = []string{
"Error extracting sts from embedded url response",
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
"video is too short to process",
"This video contains content from",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"giving up after 0 fragment retries",
"Sign in to confirm your age",
"Playback on other websites has been disabled by the video owner",
"uploader has not made this video available in your country",
"This video has been removed by the uploader",
}
type SyncFlags struct {
TakeOverExistingChannel bool
SkipSpaceCheck bool
SyncUpdate bool
SingleRun bool
RemoveDBUnpublished bool
UpgradeMetadata bool
DisableTransfers bool
QuickSync bool
MaxTries int
Refill int
Limit int
Status string
SecondaryStatus string
ChannelID string
SyncFrom int64
SyncUntil int64
ConcurrentJobs int
VideosLimit int
MaxVideoSize int
MaxVideoLength time.Duration
}
// VideosToSync dynamically figures out how many videos should be synced for a given subs count if nothing was otherwise specified
func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
if f.VideosLimit > 0 {
return f.VideosLimit
}
defaultVideosToSync := map[int]int{
10000: 1000,
5000: 500,
1000: 400,
800: 250,
600: 200,
200: 80,
100: 50,
1: 10,
}
videosToSync := 0
for s, r := range defaultVideosToSync {
if int(totalSubscribers) >= s && r > videosToSync {
videosToSync = r
}
}
return videosToSync
}
func (f *SyncFlags) IsSingleChannelSync() bool {
return f.ChannelID != ""
}
type VideoStatus struct {
ChannelID string
VideoID string
Status string
ClaimID string
ClaimName string
FailureReason string
Size *int64
MetaDataVersion uint
IsTransferred *bool
}
const (
StatusPending = "pending" // waiting for permission to sync
StatusPendingEmail = "pendingemail" // permission granted but missing email
StatusQueued = "queued" // in sync queue. will be synced soon
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
StatusSyncing = "syncing" // syncing now
StatusSynced = "synced" // done
StatusWipeDb = "pendingdbwipe" // in sync queue. lbryum database will be pruned
StatusFailed = "failed"
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
)
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
const LatestMetadataVersion = 2
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
VideoStatusTranferFailed = "transferfailed"
)
const (
TransferStateNotTouched = iota
TransferStatePending
TransferStateComplete
TransferStateManual
)
type AwsConfigs struct {
AwsS3ID string
AwsS3Secret string
AwsS3Region string
AwsS3Bucket string
}
func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
return &aws.Config{
Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
Region: &a.AwsS3Region,
}
}
+20
View File
@@ -0,0 +1,20 @@
package shared
import (
"testing"
"gotest.tools/assert"
)
func TestSyncFlags_VideosToSync(t *testing.T) {
f := SyncFlags{}
assert.Equal(t, f.VideosToSync(0), 0)
assert.Equal(t, f.VideosToSync(1), 10)
assert.Equal(t, f.VideosToSync(5), 10)
assert.Equal(t, f.VideosToSync(10), 10)
assert.Equal(t, f.VideosToSync(101), 50)
assert.Equal(t, f.VideosToSync(500), 80)
assert.Equal(t, f.VideosToSync(21000), 1000)
f.VideosLimit = 1337
assert.Equal(t, f.VideosToSync(21), 1337)
}
+6 -3
View File
@@ -2,9 +2,10 @@ package sources
import (
"strings"
"sync"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/ytsync/v5/namer"
)
type SyncSummary struct {
@@ -12,7 +13,9 @@ type SyncSummary struct {
ClaimName string
}
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer) (*SyncSummary, error) {
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer, walletLock *sync.RWMutex) (*SyncSummary, error) {
walletLock.RLock()
defer walletLock.RUnlock()
for {
name := namer.GetNextName(title)
response, err := daemon.StreamCreate(name, filename, amount, options)
+560 -125
View File
@@ -1,32 +1,45 @@
package sources
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"math"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/abadojack/whatlanggo"
"github.com/lbryio/ytsync/v5/downloader"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/ytsync/v5/shared"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
"gopkg.in/vansante/go-ffprobe.v2"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
)
type YoutubeVideo struct {
@@ -36,20 +49,23 @@ type YoutubeVideo struct {
playlistPosition int64
size *int64
maxVideoSize int64
maxVideoLength float64
maxVideoLength time.Duration
publishedAt time.Time
dir string
youtubeInfo *youtube.Video
youtubeInfo *ytdl.YtdlVideo
youtubeChannelID string
tags []string
awsConfig aws.Config
thumbnailURL string
lbryChannelID string
mocked bool
walletLock *sync.RWMutex
stopGroup *stop.Group
pool *ip_manager.IPPool
progressBars *mpb.Progress
progressBarWg *sync.WaitGroup
}
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "film & animation",
"2": "autos & vehicles",
@@ -85,22 +101,25 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
// youtube-dl returns times in local timezone sometimes. this could break in the future
// maybe we can file a PR to choose the timezone we want from youtube-dl
return &YoutubeVideo{
id: videoData.Id,
title: videoData.Snippet.Title,
description: videoData.Snippet.Description,
id: videoData.ID,
title: videoData.Title,
description: videoData.Description,
playlistPosition: playlistPosition,
publishedAt: publishedAt,
publishedAt: videoData.UploadDateForReal,
dir: directory,
youtubeInfo: videoData,
awsConfig: awsConfig,
mocked: false,
youtubeChannelID: videoData.Snippet.ChannelId,
}
youtubeChannelID: videoData.ChannelID,
stopGroup: stopGroup,
pool: pool,
}, nil
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
@@ -108,6 +127,8 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
awsConfig: awsConfig,
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
pool: pool,
}
}
@@ -158,42 +179,47 @@ func (v *YoutubeVideo) getFullPath() string {
}
func (v *YoutubeVideo) getAbbrevDescription() string {
maxLines := 10
maxLength := 2800
description := strings.TrimSpace(v.description)
if strings.Count(description, "\n") < maxLines {
return description
}
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
if v.lbryChannelID == khanAcademyClaimID {
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
}
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
if len(description) > maxLength {
description = description[:maxLength]
}
return description + "\n..." + additionalDescription
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4")
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
func checkCookiesIntegrity() error {
fi, err := os.Stat("cookies.txt")
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
return errors.Err(err)
}
if fi.Size() == 0 {
log.Errorf("cookies were cleared out. Attempting a restore from cookies-backup.txt")
input, err := ioutil.ReadFile("cookies-backup.txt")
if err != nil {
return errors.Err(err)
}
err = ioutil.WriteFile("cookies.txt", input, 0644)
if err != nil {
return errors.Err(err)
}
}
return nil
}
func (v *YoutubeVideo) download() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("download").Add(time.Since(start))
}(start)
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
err := os.Mkdir(v.videoDir(), 0777)
if err != nil && !strings.Contains(err.Error(), "file exists") {
return errors.Wrap(err, 0)
}
@@ -205,77 +231,420 @@ func (v *YoutubeVideo) download() error {
log.Debugln(v.id + " already exists at " + videoPath)
return nil
}
qualities := []string{
"1080",
"720",
"480",
"360",
}
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
if dur.Hours() > 1 { //for videos longer than 1 hour only sync up to 720p
qualities = []string{
"720",
"480",
"360",
}
}
videoUrl := "https://www.youtube.com/watch?v=" + v.id
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
metadataPath := path.Join(logUtils.GetVideoMetadataDir(), v.id+".info.json")
_, err = os.Stat(metadataPath)
if err != nil {
if os.IsNotExist(err) {
return errors.Err("metadata information for video %s is missing! Why?", v.id)
}
return errors.Err(err)
}
codec := []string{"H.264"}
ext := []string{"mp4"}
metadata, err := parseVideoMetadata(metadataPath)
//Filter requires a [] interface{}
codecFilter := make([]interface{}, len(codec))
for i, v := range codec {
codecFilter[i] = v
err = checkCookiesIntegrity()
if err != nil {
return err
}
//Filter requires a [] interface{}
extFilter := make([]interface{}, len(ext))
for i, v := range ext {
extFilter[i] = v
ytdlArgs := []string{
"--no-progress",
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
"--postprocessor-args",
"ffmpeg:-movflags faststart",
"--abort-on-unavailable-fragment",
"--fragment-retries",
"1",
"--cookies",
"cookies.txt",
"--extractor-args",
"youtube:player_client=android",
//"--concurrent-fragments",
//"2",
"--load-info-json",
metadataPath,
}
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
if len(formats) == 0 {
return errors.Err("no compatible format available for this video")
userAgent := []string{"--user-agent", downloader.ChromeUA}
if v.maxVideoSize > 0 {
ytdlArgs = append(ytdlArgs,
"--max-filesize",
fmt.Sprintf("%dM", v.maxVideoSize),
)
}
maxRetryAttempts := 5
isLengthLimitSet := v.maxVideoLength > 0.01
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
return errors.Err("video is too long to process")
if v.maxVideoLength > 0 {
ytdlArgs = append(ytdlArgs,
"--match-filter",
fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
)
}
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
formatIndex := i
if i == maxRetryAttempts-1 {
formatIndex = len(formats) - 1
var sourceAddress string
for {
sourceAddress, err = v.pool.GetIP(v.id)
if err != nil {
if errors.Is(err, ip_manager.ErrAllThrottled) {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
default:
time.Sleep(ip_manager.IPCooldownPeriod)
continue
}
} else {
return err
}
}
var downloadedFile *os.File
downloadedFile, err = os.Create(videoPath)
break
}
defer v.pool.ReleaseIP(sourceAddress)
ytdlArgs = append(ytdlArgs,
"--source-address",
sourceAddress,
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
)
//speedThrottleRetries := 3
for i := 0; i < len(qualities); i++ {
quality := qualities[i]
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
argsWithFilters = append(argsWithFilters, userAgent...)
//if speedThrottleRetries > 0 {
// speedThrottleRetries--
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
//}
cmd := exec.Command("yt-dlp", argsWithFilters...)
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
err = videoInfo.Download(formats[formatIndex], downloadedFile)
_ = downloadedFile.Close()
stdout, err := cmd.StdoutPipe()
if err != nil {
//delete the video and ignore the error
_ = v.delete()
return errors.Err(err.Error())
return errors.Err(err)
}
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
dlStopGrp := stop.New()
ticker := time.NewTicker(400 * time.Millisecond)
go v.trackProgressBar(argsWithFilters, ticker, metadata, dlStopGrp, sourceAddress)
//ticker2 := time.NewTicker(10 * time.Second)
//v.monitorSlowDownload(ticker, dlStopGrp, sourceAddress, cmd)
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
err = cmd.Wait()
//stop the progress bar
ticker.Stop()
dlStopGrp.Stop()
if err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
if i == (len(qualities) - 1) {
return errors.Err(string(errorLog))
}
continue //this bypasses the yt throttling IP redistribution... TODO: don't
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
i-- //do not lower quality when trying a different user agent
userAgent = []string{downloader.GoogleBotUA}
log.Infof("trying different user agent for video %s", v.ID())
continue
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
// i-- //do not lower quality when we're retrying a throttled download
// continue
}
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete("does not pass filter duration")
return errors.Err("video is too long to process")
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete("File is larger than max-filesize")
return errors.Err("the video is too big to sync, skipping for now")
}
if string(errorLog) != "" {
if strings.Contains(string(errorLog), "HTTP Error 429") {
v.pool.SetThrottled(sourceAddress)
}
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete("due to error")
return errors.Err(string(errorLog))
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
err = os.Chmod(v.getFullPath(), 0777)
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
isVideoSizeLimitSet := v.maxVideoSize > 0
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
//delete the video and ignore the error
_ = v.delete()
err = errors.Err("file is too big and there is no other format available")
} else {
break
break
}
return nil
}
func (v *YoutubeVideo) monitorSlowDownload(ticker *time.Ticker, stop *stop.Group, address string, cmd *exec.Cmd) {
count := 0
lastSize := int64(0)
for {
select {
case <-stop.Ch():
return
case <-ticker.C:
size, err := logUtils.DirSize(v.videoDir())
if err != nil {
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
continue
}
delta := size - lastSize
avgSpeed := delta / 10
if avgSpeed < 200*1024 { //200 KB/s
count++
} else {
count--
}
if count > 3 {
err := cmd.Process.Signal(syscall.SIGKILL)
if err != nil {
log.Errorf("failure in killing slow download: %s", errors.Err(err))
return
}
}
}
}
return errors.Err(err)
}
func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.Ticker, metadata *ytMetadata, done *stop.Group, sourceAddress string) {
v.progressBarWg.Add(1)
go func() {
defer v.progressBarWg.Done()
//get size of the video before downloading
cmd := exec.Command("yt-dlp", append(argsWithFilters, "-s")...)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
return
}
if err := cmd.Start(); err != nil {
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
return
}
outLog, _ := ioutil.ReadAll(stdout)
err = cmd.Wait()
output := string(outLog)
parts := strings.Split(output, ": ")
if len(parts) != 3 {
log.Errorf("couldn't parse audio and video parts from the output (%s)", output)
return
}
formats := strings.Split(parts[2], "+")
if len(formats) != 2 {
log.Errorf("couldn't parse formats from the output (%s)", output)
return
}
log.Debugf("'%s'", output)
videoFormat := formats[0]
audioFormat := strings.Replace(formats[1], "\n", "", -1)
videoSize := 0
audioSize := 0
if metadata != nil {
for _, f := range metadata.Formats {
if f.FormatID == videoFormat {
videoSize = f.Filesize
}
if f.FormatID == audioFormat {
audioSize = f.Filesize
}
}
}
log.Debugf("(%s) - videoSize: %d (%s), audiosize: %d (%s)", v.id, videoSize, videoFormat, audioSize, audioFormat)
bar := v.progressBars.AddBar(int64(videoSize+audioSize),
mpb.PrependDecorators(
decor.CountersKibiByte("% .2f / % .2f "),
// simple name decorator
decor.Name(fmt.Sprintf("id: %s src-ip: (%s)", v.id, sourceAddress)),
// decor.DSyncWidth bit enables column width synchronization
decor.Percentage(decor.WCSyncSpace),
),
mpb.AppendDecorators(
decor.EwmaETA(decor.ET_STYLE_GO, 90),
decor.Name(" ] "),
decor.EwmaSpeed(decor.UnitKiB, "% .2f ", 60),
decor.OnComplete(
// ETA decorator with ewma age of 60
decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
),
),
mpb.BarRemoveOnComplete(),
)
defer func() {
bar.Completed()
bar.Abort(true)
}()
for {
select {
case <-done.Ch():
return
case <-ticker.C:
size, err := logUtils.DirSize(v.videoDir())
if err != nil {
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
return
}
bar.SetCurrent(size)
if size > int64(videoSize+audioSize) {
bar.SetTotal(size+2048, false)
}
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
}
}
}()
}
type ytMetadata struct {
ID string `json:"id"`
Title string `json:"title"`
Formats []struct {
Asr int `json:"asr"`
Filesize int `json:"filesize"`
FormatID string `json:"format_id"`
FormatNote string `json:"format_note"`
Fps interface{} `json:"fps"`
Height interface{} `json:"height"`
Quality int `json:"quality"`
Tbr float64 `json:"tbr"`
URL string `json:"url"`
Width interface{} `json:"width"`
Ext string `json:"ext"`
Vcodec string `json:"vcodec"`
Acodec string `json:"acodec"`
Abr float64 `json:"abr,omitempty"`
DownloaderOptions struct {
HTTPChunkSize int `json:"http_chunk_size"`
} `json:"downloader_options,omitempty"`
Container string `json:"container,omitempty"`
Format string `json:"format"`
Protocol string `json:"protocol"`
HTTPHeaders struct {
UserAgent string `json:"User-Agent"`
AcceptCharset string `json:"Accept-Charset"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Vbr float64 `json:"vbr,omitempty"`
} `json:"formats"`
Thumbnails []struct {
Height int `json:"height"`
URL string `json:"url"`
Width int `json:"width"`
Resolution string `json:"resolution"`
ID string `json:"id"`
} `json:"thumbnails"`
Description string `json:"description"`
UploadDate string `json:"upload_date"`
Uploader string `json:"uploader"`
UploaderID string `json:"uploader_id"`
UploaderURL string `json:"uploader_url"`
ChannelID string `json:"channel_id"`
ChannelURL string `json:"channel_url"`
Duration int `json:"duration"`
ViewCount int `json:"view_count"`
AverageRating float64 `json:"average_rating"`
AgeLimit int `json:"age_limit"`
WebpageURL string `json:"webpage_url"`
Categories []string `json:"categories"`
Tags []interface{} `json:"tags"`
IsLive interface{} `json:"is_live"`
LikeCount int `json:"like_count"`
DislikeCount int `json:"dislike_count"`
Channel string `json:"channel"`
Extractor string `json:"extractor"`
WebpageURLBasename string `json:"webpage_url_basename"`
ExtractorKey string `json:"extractor_key"`
Playlist interface{} `json:"playlist"`
PlaylistIndex interface{} `json:"playlist_index"`
Thumbnail string `json:"thumbnail"`
DisplayID string `json:"display_id"`
Format string `json:"format"`
FormatID string `json:"format_id"`
Width int `json:"width"`
Height int `json:"height"`
Resolution interface{} `json:"resolution"`
Fps int `json:"fps"`
Vcodec string `json:"vcodec"`
Vbr float64 `json:"vbr"`
StretchedRatio interface{} `json:"stretched_ratio"`
Acodec string `json:"acodec"`
Abr float64 `json:"abr"`
Ext string `json:"ext"`
Fulltitle string `json:"fulltitle"`
Filename string `json:"_filename"`
}
func parseVideoMetadata(metadataPath string) (*ytMetadata, error) {
f, err := os.Open(metadataPath)
if err != nil {
return nil, errors.Err(err)
}
// defer the closing of our jsonFile so that we can parse it later on
defer f.Close()
// read our opened jsonFile as a byte array.
byteValue, _ := ioutil.ReadAll(f)
// we initialize our Users array
var m ytMetadata
// we unmarshal our byteArray which contains our
// jsonFile's content into 'users' which we defined above
err = json.Unmarshal(byteValue, &m)
if err != nil {
return nil, errors.Err(err)
}
return &m, nil
}
func (v *YoutubeVideo) videoDir() string {
return v.dir + "/" + v.id
return path.Join(v.dir, v.id)
}
func (v *YoutubeVideo) getDownloadedPath() (string, error) {
files, err := ioutil.ReadDir(v.videoDir())
log.Infoln(v.videoDir())
@@ -290,20 +659,20 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) {
continue
}
if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
return v.videoDir() + "/" + f.Name(), nil
return path.Join(v.videoDir(), f.Name()), nil
}
}
return "", errors.Err("could not find any downloaded videos")
}
func (v *YoutubeVideo) delete() error {
func (v *YoutubeVideo) delete(reason string) error {
videoPath, err := v.getDownloadedPath()
if err != nil {
log.Errorln(err)
return err
}
err = os.Remove(videoPath)
log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath)
if err != nil {
err = errors.Prefix("delete error", err)
@@ -315,14 +684,20 @@ func (v *YoutubeVideo) delete() error {
}
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
if thumbnail.Width == 0 {
return errors.Err("default youtube thumbnail found")
}
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
return err
}
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("publish").Add(time.Since(start))
}(start)
languages, locations, tags := v.getMetadata()
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
@@ -335,7 +710,17 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
}
}
urlsRegex := regexp.MustCompile(`(?m) ?(f|ht)(tp)(s?)(://)(.*)[.|/](.*)`)
descriptionSample := urlsRegex.ReplaceAllString(v.description, "")
info := whatlanggo.Detect(descriptionSample)
info2 := whatlanggo.Detect(v.title)
if info.IsReliable() && info.Lang.Iso6391() != "" {
language := info.Lang.Iso6391()
languages = []string{language}
} else if info2.IsReliable() && info2.Lang.Iso6391() != "" {
language := info2.Lang.Iso6391()
languages = []string{language}
}
options := jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: &v.title,
@@ -345,6 +730,9 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
ThumbnailURL: &v.thumbnailURL,
Tags: tags,
Locations: locations,
FundingAccountIDs: []string{
params.DefaultAccount,
},
},
Fee: fee,
License: util.PtrToString("Copyrighted (contact publisher)"),
@@ -355,7 +743,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
if err != nil {
return nil, err
}
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer, v.walletLock)
}
func (v *YoutubeVideo) Size() *int64 {
@@ -368,33 +756,63 @@ type SyncParams struct {
ChannelID string
MaxVideoSize int
Namer *namer.Namer
MaxVideoLength float64
Fee *sdk.Fee
MaxVideoLength time.Duration
Fee *shared.Fee
DefaultAccount string
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex, pbWg *sync.WaitGroup, pb *mpb.Progress) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize)
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
v.progressBars = pb
v.progressBarWg = pbWg
if reprocess && existingVideoData != nil && existingVideoData.Published {
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, err
return summary, errors.Prefix("upgrade failed", err)
}
return v.downloadAndPublish(daemon, params)
}
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
var err error
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
minDuration := 7 * time.Second
if v.youtubeInfo.IsLive == true {
return nil, errors.Err("video is a live stream and hasn't completed yet")
}
if dur > v.maxVideoLength {
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
return nil, errors.Err("video is too long to process")
}
if dur < minDuration {
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
return nil, errors.Err("video is too short to process")
}
for {
err = v.download()
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
continue
} else if err != nil {
return nil, errors.Prefix("download error", err)
}
break
}
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelFn()
data, err := ffprobe.ProbeURL(ctx, v.getFullPath())
if err != nil {
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
fallBackErr := v.fallbackDownload()
if fallBackErr != nil {
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
return nil, errors.Prefix("download error", err) //return original error
log.Errorf("failure in probing downloaded video: %s", err.Error())
} else {
if data.Format.Duration() < minDuration {
return nil, errors.Err("video is too short to process")
}
}
log.Debugln("Downloaded " + v.id)
err = v.triggerThumbnailSave()
if err != nil {
return nil, errors.Prefix("thumbnail error", err)
@@ -403,7 +821,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
summary, err := v.publish(daemon, params)
//delete the video in all cases (and ignore the error)
_ = v.delete()
_ = v.delete("finished download and publish")
return summary, errors.Prefix("publish error", err)
}
@@ -413,31 +831,37 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
locations = nil
tags = nil
if !v.mocked {
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}
/*
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
v.youtubeInfo.Snippet.DefaultLanguage = "he"
}
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}*/
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
locations = []jsonrpc.Location{{
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
}}
}
tags = v.youtubeInfo.Snippet.Tags
}*/
tags = v.youtubeInfo.Tags
}
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
if err != nil {
log.Errorln(err.Error())
}
if !v.mocked {
tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
for _, category := range v.youtubeInfo.Categories {
tags = append(tags, youtubeCategories[category])
}
}
return languages, locations, tags
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
if err != nil {
return nil, errors.Err(err)
}
@@ -455,8 +879,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
if v.mocked {
return nil, errors.Err("could not find thumbnail for mocked video")
}
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
} else {
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
}
@@ -467,10 +891,17 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
videoSize = uint64(existingVideoData.Size)
} else {
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
return nil, errors.Err("the video must be republished as we can't get the right size")
if !v.mocked {
_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
if err != nil {
return nil, errors.Err(err)
}
return v.downloadAndPublish(daemon, params)
}
return nil, errors.Prefix("the video must be republished as we can't get the right size and it doesn't exist on youtube anymore", err)
}
}
v.size = util.PtrToInt64(int64(videoSize))
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
@@ -489,6 +920,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
ThumbnailURL: &thumbnailURL,
Languages: languages,
Locations: locations,
FundingAccountIDs: []string{
params.DefaultAccount,
},
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),
@@ -498,11 +932,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
Fee: fee,
}
v.walletLock.RLock()
defer v.walletLock.RUnlock()
if v.mocked {
start := time.Now()
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: streamCreateOptions,
FileSize: &videoSize,
})
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
if err != nil {
return nil, err
}
@@ -513,15 +951,11 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
}, nil
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
streamCreateOptions.ClaimCreateOptions.Title = &v.title
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
start := time.Now()
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
ClearLanguages: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
@@ -529,6 +963,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
StreamCreateOptions: streamCreateOptions,
FileSize: &videoSize,
})
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
if err != nil {
return nil, err
}
+53
View File
@@ -0,0 +1,53 @@
package sources
import (
"regexp"
"testing"
"github.com/abadojack/whatlanggo"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestLanguageDetection(t *testing.T) {
description := `Om lättkränkta muslimer, och den bristande logiken i vad som anses vara att vanära profeten. Från Moderata riksdagspolitikern Hanif Balis podcast "God Ton", avsnitt 108, från oktober 2020, efter terrordådet där en fransk lärare fick huvudet avskuret efter att undervisat sin mångkulturella klass om frihet.`
info := whatlanggo.Detect(description)
logrus.Infof("confidence: %.2f", info.Confidence)
assert.True(t, info.IsReliable())
assert.True(t, info.Lang.Iso6391() != "")
assert.Equal(t, "sv", info.Lang.Iso6391())
description = `🥳週四直播 | 晚上來開個賽車🔰歡迎各位一起來玩! - PonPonLin蹦蹦林`
info = whatlanggo.Detect(description)
logrus.Infof("confidence: %.2f", info.Confidence)
assert.True(t, info.IsReliable())
assert.True(t, info.Lang.Iso6391() != "")
assert.Equal(t, "zh", info.Lang.Iso6391())
description = `成為這個頻道的會員並獲得獎勵:
https://www.youtube.com/channel/UCOQFrooz-YGHjYb7s3-MrsQ/join
_____________________________________________
想聽我既音樂作品可以去下面LINK
streetvoice 街聲:
https://streetvoice.com/CTLam331/
_____________________________________________
想學結他、鋼琴
有關音樂制作工作
都可以搵我~
大家快D訂閱喇
不定期出片
Website: http://ctlam331.wixsite.com/ctlamusic
FB PAGE:https://www.facebook.com/ctlam331
IG:ctlamusic`
urlsRegex := regexp.MustCompile(`(?m) ?(f|ht)(tp)(s?)(://)(.*)[.|/](.*)`)
descriptionSample := urlsRegex.ReplaceAllString(description, "")
info = whatlanggo.Detect(descriptionSample)
logrus.Infof("confidence: %.2f", info.Confidence)
assert.True(t, info.IsReliable())
assert.True(t, info.Lang.Iso6391() != "")
assert.Equal(t, "zh", info.Lang.Iso6391())
}
-152
View File
@@ -1,152 +0,0 @@
import os
import sys
from decimal import Decimal
from bitcoinrpc.authproxy import AuthServiceProxy
from lbryum.wallet import Wallet, WalletStorage
from lbryum.commands import known_commands, Commands
from lbryum.simple_config import SimpleConfig
from lbryum.blockchain import get_blockchain
from lbryum.network import Network
def get_lbrycrdd_connection_string(wallet_conf):
settings = {"username": "rpcuser",
"password": "rpcpassword",
"rpc_port": 9245}
if wallet_conf and os.path.exists(wallet_conf):
with open(wallet_conf, "r") as conf:
conf_lines = conf.readlines()
for l in conf_lines:
if l.startswith("rpcuser="):
settings["username"] = l[8:].rstrip('\n')
if l.startswith("rpcpassword="):
settings["password"] = l[12:].rstrip('\n')
if l.startswith("rpcport="):
settings["rpc_port"] = int(l[8:].rstrip('\n'))
rpc_user = settings["username"]
rpc_pass = settings["password"]
rpc_port = settings["rpc_port"]
rpc_url = "127.0.0.1"
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
class LBRYumWallet(object):
def __init__(self, lbryum_path):
self.config = SimpleConfig()
self.config.set_key('chain', 'lbrycrd_main')
self.storage = WalletStorage(lbryum_path)
self.wallet = Wallet(self.storage)
self.cmd_runner = Commands(self.config, self.wallet, None)
if not self.wallet.has_seed():
seed = self.wallet.make_seed()
self.wallet.add_seed(seed, "derp")
self.wallet.create_master_keys("derp")
self.wallet.create_main_account()
self.wallet.update_password("derp", "")
self.network = Network(self.config)
self.blockchain = get_blockchain(self.config, self.network)
print self.config.get('chain'), self.blockchain
self.wallet.storage.write()
def command(self, command_name, *args, **kwargs):
cmd_runner = Commands(self.config, self.wallet, None)
cmd = known_commands[command_name]
func = getattr(cmd_runner, cmd.name)
return func(*args, **kwargs)
def generate_address(self):
address = self.wallet.create_new_address()
self.wallet.storage.write()
return address
class LBRYcrd(object):
def __init__(self, lbrycrdd_path):
self.lbrycrdd_conn_str = get_lbrycrdd_connection_string(lbrycrdd_path)
def __call__(self, method, *args, **kwargs):
return self.rpc(method)(*args, **kwargs)
def rpc(self, method):
return AuthServiceProxy(self.lbrycrdd_conn_str, service_name=method)
def get_wallet_path():
cwd = os.getcwd()
wallet_path = os.path.join(cwd, "wallet.json")
if not os.path.exists(wallet_path):
return wallet_path
i = 1
while True:
wallet_path = os.path.join(cwd, "wallet_%i.json" % i)
if not os.path.exists(wallet_path):
return wallet_path
i += 1
def coin_chooser(lbrycrdd, amount, fee=0.001):
def iter_txis():
unspent = lbrycrdd("listunspent")
unspent = sorted(unspent, key=lambda x: x['amount'], reverse=True)
spendable = Decimal(0.0)
for txi in unspent:
if spendable >= amount:
break
else:
spendable += txi['amount']
yield txi
if spendable < amount:
print spendable, amount
raise Exception("Not enough funds")
coins = list(iter(iter_txis()))
total = sum(c['amount'] for c in coins)
change = Decimal(total) - Decimal(amount) - Decimal(fee)
if change < 0:
raise Exception("Not enough funds")
if change:
change_address = lbrycrdd("getnewaddress")
else:
change_address = None
print "Total: %f, amount: %f, change: %f" % (total, amount, change)
return coins, change, change_address
def get_raw_tx(lbrycrdd, addresses, coins, amount, change, change_address):
txi = [{'txid': c['txid'], 'vout': c['vout']} for c in coins]
txo = {address: float(amount) for address in addresses}
if change_address:
txo[change_address] = float(change)
return lbrycrdd("createrawtransaction", txi, txo)
def main(count, value=None, lbryum_path=None, lbrycrdd_path=None):
count = int(count)
lbryum_path = lbryum_path or get_wallet_path()
if sys.platform == "darwin":
default_lbrycrdd = os.path.join(os.path.expanduser("~"),
"Library/Application Support/lbrycrd/lbrycrd.conf")
else:
default_lbrycrdd = os.path.join(os.path.expanduser("~"), ".lbrycrd/lbrycrd.conf")
lbrycrdd_path = lbrycrdd_path or default_lbrycrdd
l = LBRYcrd(lbrycrdd_path=lbrycrdd_path)
s = LBRYumWallet(lbryum_path)
value = value or 1.0
value = Decimal(value)
coins, change, change_address = coin_chooser(l, count * value)
addresses = [s.generate_address() for i in range(count)]
raw_tx = get_raw_tx(l, addresses, coins, value, change, change_address)
signed = l("signrawtransaction", raw_tx)['hex']
txid = l("sendrawtransaction", signed)
print txid
if __name__ == "__main__":
args = sys.argv[1:]
main(*args)
@@ -1,4 +1,4 @@
package tagsManager
package tags_manager
import (
"regexp"
@@ -71,6 +71,8 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
return sanitizedTags, nil
}
const TagMaxLength = 50
func normalizeTag(t string) (string, error) {
t = strings.ToLower(t)
multipleSpaces := regexp.MustCompile(`\s{2,}`)
@@ -87,7 +89,10 @@ func normalizeTag(t string) (string, error) {
if weirdChars.MatchString(t) {
log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
return "", nil
}
if len(t) > TagMaxLength {
log.Debugf("tag '%s' is too long, skipping\n", t)
return "", nil
}
return t, nil
}
@@ -140,19 +145,417 @@ func (ts *tagsSanitizer) add() {
}
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
anupjanakpur = "UC_5tRfC4L2AbTz6mj6vZrKw"
PraveenMedia = "UC_fjE70lKNwM9AKIofQv-bA"
MisteriosDesvelados = "UC-FzxivscjYzonBDXgX2GLg"
kaipenchery = "UC-MU4K3Ghl-IdEX4J68tnTA"
Dhinchakpooja = "UC-stzLwoQF_R8Vnfb28d7Lg"
Karolajn = "UC-vYDJQiabg9BK8XTDPFspg"
thefantasio974 = "UC0GHeUcPxfNEZqbFyxy2frQ"
EduPrimitivo = "UC0odmP6ffEw3iVwTEyt-UKA"
isupportanupriyapatel = "UC0sxOzdHnmauMdOra0cxXVg"
theconfusedindian = "UC12ZPYxQbMA1loZE1OA609w"
guriaarts = "UC1fHp166o1Hd024fg3-d-jg"
minutodafisica = "UC1lUEcHrQwbusC5ext1himg"
khabir = "UC1yzNCXXk1h_tHgsqChE8mw"
OMapadaMinaMara = "UC2WMUPTbxQQ9hXq2_uutspg"
oliverjanich = "UC3cmEfpy4XED7YYEe69nIMA"
shyamsadhu = "UC3XAT8oBjL2RaqfblESHW_A"
EbaduRahmantech = "UC4950gpY6Qw1lCAcN6gNWwQ"
_1975oles = "UC4bGQWN4C8idymxlZxw77oA"
_8dsongsbollywood = "UC4nH7zmw41lRDbaiIddumjw"
EYESTV = "UC50CIbyHMydEEzEhV1ZNEBw"
Nono = "UC5yufoRPJy-1e1pR73UTxFQ"
lichtle = "UC6n_2v7YjwZ65F9h77nZPOA"
elabuelokraken = "UC7iWk2xziMR4hs_NzfsYytw"
cidvela = "UCaVOx5GCcSi2ELjWUdf05PA"
jayaskumar = "UCax9CJCQ6aY0bntP3mcIWKQ"
canaldarippi = "UCB_fba7yYMwa91F7rijBsVA"
FunMusicClassical = "UCB_X256IN7QiBtDkueo_6QQ"
minutodaterra = "UCB0zinWfy-dS_NqcOINYo3A"
franciscoalves = "UCbAm6YcZGk04obnFb-LuOSA"
criptomonedastv = "UCbK6Awel1-o-9JDVMFq808Q"
lux77 = "UCBYm4l3NX352goFtjSgZ7jA"
MrLuckyOficial = "UCEbMhGhZ1JoOYKgbgVyeQaA"
KotneKit = "UCf_1BQz3T151Eph0Sarj3Yw"
jaysgaming = "UCFAkedtc3jjDZQqqIULxqWg"
iamdineshthakur = "UCg0VInmUZoSdHD-DNmjifBA"
Musicreationz = "UCGJEtZB0Nj3rT3zZfyfjIZQ"
lafenixtv = "UCHAx8o0jj-8L_sqR5QM1Cwg"
guardeiafe = "UCHWjmG-Ce93VNcNZMMUyGQQ"
KHANEKIKHUSHBOO = "UCI8DRKcrfrHklRqSJHPhOwQ"
AjayGamingYT = "UCi9r5igvQblbI0PhJFj-IDg"
SzekelyVegan = "UCiqvKH1Ib_VPwh9bQf_hlXQ"
barzoy888 = "UCIYF1orTg4nDvv29ORoKXyA"
_7playstudiopersian = "UCJ3QUKcd9yOhiJzZBR2zT3Q"
eazypurple = "UCJ6Hd8S39g7pe65A6k42NZw"
GennadyM = "UCjZlOrmC7hi_1kMA6BKZXeQ"
GTGujarati = "UCKnYZmYFdFX4hP09Hh8k8jw"
annapavlidou = "UCKPSHyCFIDtMqzbJ_Qx-7aw"
Tamilmemesclub = "UCL_DPW38jcSGWy0Fu0A_hCg"
QuickReactionTeam = "UCl--LVcANGKHJYN7O8uY6OA"
rodrigojoseoficial = "UClMsSb57Jvexl1a0a2c2_CQ"
AzadChaiwala = "UCM5QNdoIefx6eumjPk8ZTMw"
lafenixextras = "UCM7zougvyPtfd9zw5HmAGlg"
JustSawAGhost = "UCmQYXE03n6qlF9FhtAPztcw"
mirchibangla = "UCmzj6hXrPZ_AwIZ8lgo-HuQ"
marsaguirre = "UCN3OMvdU7ySvE6meGjo9omg"
JustGoool = "UCn7kTV_syU_rI0jANxrpaFg"
nossocanal = "UCNDb9jdx5Jz-C7hWJv3y6Fw"
CSNN = "UCnfn-8PJbjYFAJ1fqzMdcew"
pedronedved = "UCNhb2Fz3sVLZcMb_arm1cMQ"
famoosh = "UCnQU2QZLVG1wxHoy_9LZucA"
Cenoritas = "UCo-VSGoYy_4IjXF9u4QqTiw"
fantabobgames = "UCo0U1tbk3YbqiLDhkeWOviQ"
thesoundofsoul = "UCOey6R7Ktnil8RZrY_xJYiA"
SaurabhMishraJournalist = "UCoHVCBoSfUOlQVa80dlN60g"
YoYoHoneySinghlive = "UCOZTp3nGj39-snzo4QSdduA"
CINEGLITZ = "UCP_p8JkLOwPcGAnxN1iVhZA"
lovetreta = "UCPi3uuGrh2mxnZJMfRhcjiw"
canalvendimeusofa = "UCpk58NDdaKdX0QiiA2e79tg"
CARIBEANVIDEOS = "UCPynGkfzH35RKyXRccCgzNg"
Dimon = "UCpZsZZ6KEqCHeAJ_Y0gEbyQ"
Anantvijaysoni = "UCQ3NeFKF2yo70xxBBUDeXXQ"
canalilhadebarbados = "UCR5L_Q8Tiljy7WKEQOrGKbg"
SotomayorTV2 = "UCR70CjRHxQilfEUgBBYpDhg"
Recipesarab = "UCRbMDfkH_bPUjQsv5dHcFQA"
_4ak10 = "UCsbrIjDPPXuVlWApYTnQECA"
Top50 = "UCSjXR8uvU4PijMM_kgyVQfw"
SoumyajitPyne = "UCSJXwgF3SePNfyojYG3yt6Q"
SneksShow = "UCsqZSkVccnyIxSE6pZuj13g"
casadosaber = "UCtvvTFp0XANyllOdmzZr9VQ"
Entarexyt = "UCuDkq6yAb5zdM2zIRG6VRDw"
UCurU6GLM4ggcLtWWAOTzlYw = "UCurU6GLM4ggcLtWWAOTzlYw"
promining = "UCviqKxlMnZqBipTzwixkGyg"
zapkids = "UCx2uGhYa9EuCNmbL72hVKGg"
Canalokok = "UCXyCAcjoWz9SMLGJT8dR0pA"
SmoothGames = "UCY8gvG25rOZ4hWWvcQH88PQ"
LIGATFA = "UCYhiIMOlDn_HEn3zvqZCDiw"
DeHamad = "UCZ5G07Vw7IaV81InBgYLCcg"
misszizi = "UCE55WTFs4ekJ_aWCoNEapbQ"
creationshub = "UCNfELkowZPIQ-Vegmb3pIBQ"
AlicePandora = "UCScRxEtwlt082_6ThI0YJbg"
Akito0405 = "UCXHWx1teSYIKwTGiscYaC-Q"
dashcambristol = "UCD0IC8bZI-MfIutkgPHlyWA"
adrianbonillap = "UCdmY7p_kC-QN8jS7rocmCSA"
SubhamVlogs = "UCXtdoCLRlnLIW3Skix0rHaQ"
DisciplesofJesusChristJeremiahPayne = "UC_y2rVsotcQcVF7LImVGs5Q"
ModernGalaxy = "UCGISiGs_RL7Z1qfs1-GD6PA"
socofilms = "UCyDS9p6NWHpU9XbbbYLFLBw"
buriedone = "UC_lm7xXB3adOTc0T1FwyGRQ"
bitcoin = "UC-f5nPBEDyUBZz_jOPBwAfQ"
globalrashid = "UC2ldcEtbR7cFYadgrnW3B6Q"
gameofbitcoins = "UC2WKsYBxMwx7E7ENkND-fkg"
CryptoInsightsBrasil = "UC4BrnREinCBUenQZi4ZU95w"
Crypt0 = "UCdUSSt-IEUg2eq46rD7lu_g"
cryptomined = "UCGQ3XHtsH8Q9iQr9bFbgfDA"
altcoinbuzz = "UCGyqEtcGQQtXyUwvcy7Gmyg"
crypto = "UCiMgF08KQ4z-Gnu8o2BLOxA"
Crypto99 = "UCjsrdOJCAKcuBqyyqjg1cCQ"
TheCryptoLark = "UCl2oCaw8hdR_kbqyqd2klIA"
NuggetsNews = "UCLo66QVfEod0nNM_GzKNxmQ"
btckyle = "UCNCGCxxTT10aeTgUMHW5FfQ"
LouisThomas = "UCpceefaJ9vs4RYUTsO9Y3FA"
BitsBeTrippin = "UCVVWXoQfMfQVuzxcLylq9aA"
cryptocrow = "UCwsRWmIL5XKqFtdytBfeX0g"
KouSuccessLeeFX = "UC0YkP4Fg_d8y6JFwLj3MLdg"
TheSchiffReport = "UCIjuLiLHdFxYtFmWlbTGQRQ"
MRU = "UCnkEhPBMZcEO0QGu51fDFDg"
Vidello = "UCwMaWqZ6SdDpTaYOW0huELw"
kaccreative = "UC02O9ICMuwrfULSa0N6SiSw"
shecooks = "UChZYqTJkeYV2r7WETcytdPQ"
vegsource = "UClEsPxvotpTJ1Z8eu2Y97rg"
VeganGains = "UCr2eKhGzPhN5RPVk5dd5o3g"
AwesomeKnowledge = "UC_pC4T8vr-caiNDYU6PYTTA"
luckyloush = "UCbO0Oomf_jr20Wb4V2ap5Uw"
MichaelLuzzi = "UCDwLrj4DSGrw3gFvANbl_Cw"
NakedApe = "UCMOaRU-YsXVgU-WahBkZqWQ"
ParaReact = "UCuJKELjsmWTlJG0X7T4ZD_A"
dullytv = "UCv1J91Nhn7KsxMFaxKChT3w"
REDONKULAS = "UCwd_sSDZ8EQt6SEeOO2tBRA"
RedactedTonight = "UCyvaZ2RHEDrgKXz43gz7CbQ"
TonyTornado = "UCZu9AV3mrCCDpK_dy1qJRAQ"
hoodlumscrafty = "UC0cTVjYKgAnBrXQKcICyNmA"
Karmakut = "UC2B8TOklu2rSDULqAzwn5GQ"
Draxr = "UC72o3j23E1wKskBDEKlmTOw"
BigfryTV = "UC7FVdUA3SxDMfl4fLBGlADg"
CSGONews = "UC7L6NRLyldvxWukhOHABazQ"
ZedGaming = "UC7Q15H71DmB7F1iSFE1Z8LQ"
KamiVS = "UC9fh15yUcGAr8iUfQaoRRpQ"
grabthegames = "UCaJFEgY6ij05Fxgn6qtcX_g"
blueplays = "UCbjMsFlYb2NLpjS3uDzm9ow"
ImNotBonkers = "UCCyN0G77B7wnf-1AOTI3gWQ"
GunslingerMedia = "UCDjs4JoXmMzvaZyPVsvjKXw"
LeagueVoices = "UCdkonRjBLzr1Adf7Jhu1bWQ"
SirPugger = "UCelqWKTcCvBZP_1_1iYZAWA"
gamesoup = "UCGPMrF9AN_D9BrmSmMeV3hA"
ProHenis = "UCIjFoXSQ9HYbcWmRVIsH-Ow"
Dota2Divine = "UCiR9IHCurqVHpC821NVcW6g"
CrazyFoxMovies = "UCjewtQLpJEENPLbrCtb6YpA"
Larry = "UCJVdNvvuvOnthuWVQjYff2w"
hottake = "UCK24784Oqb4oYQfHL9XePNg"
Breezy = "UCKWRpZpcLKriWd1am9SHf8A"
retrorgb = "UCLPIbBCKVH2uKGm5C4sOkew"
Zer0Gamer = "UCmII34jN4rqCIsGqWkK5JEg"
nickatnyte = "UCMxYQX1zaepCgmiSmwbT39w"
DyllonStejGaming = "UCngaLL0QDbsAGYj7zsB8o_Q"
bluedrake42 = "UCNSwcDEUfIEzYdAPscXo6ZA"
Rerez = "UCoFpRCAsKfWAshvLE1bYzdw"
Op = "UCowi5kFfvGXR8NqhyE6jneQ"
kidsgamesfun = "UCqjGzmb2pMWSFYm0GzKeTEA"
GamesGlitches = "UCRj3Q06KOxAZWHaHScrkAOw"
Musikage = "UCsej4tgCoXDgVH3J7M3NMgw"
JeffyGaming = "UCTZzSNnZ43XQslejT5wFdRw"
oniblackmage = "UCUEF9XL3o8dZ6hvVf8jAi8Q"
nickatnyte2 = "UCuoTqrobMyZj0ge8LOCkiSw"
KazeEmanuar = "UCuvSqzfO_LV_QzHdmEj84SQ"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
GaminGHD = "UCW-thz5HxE-goYq8yPds1Gw"
BHGaming = "UCX4N3DioqqrugFeilxTkSIw"
Potato = "UCxPPTDNH85HZWxrgZ3FQBYA"
MikeyTaylorGaming = "UCycXj6lRWtsSqo-bZOIZePw"
Acituanbus = "UCzfx1QvKjn-BxLMLBdBGgMw"
juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
KhanAcademy = "UC4a-Gbdw7vOaccHmFo40b9g"
DON = "UCAYrPk70AePJZSaVLKrWdfQ"
shogogarcia = "UCE3yZjxDg3iI91bcNJDFnsg"
alphalifestyleacademy = "UCeggEaXtJu2domMahYMD_ig"
NileRed = "UCFhXFikryT4aFcLkLw2LBLA"
veritasium = "UCHnyfMqiRRG1u-2MsSQLbXA"
stevecronin = "UCJYawZQYwjZ76mrF_US9eNg"
jeranism = "UCS_FY5mR4g22L_E9t1D_ExQ"
MinutePhysics = "UCUHW94eEFW7hkUMVaZz4eDg"
_3Blue1Brown = "UCYO_jab_esuFRV4b17AJtAw"
Itsrucka = "UC-B2LyEZcl3avG0coKeohGQ"
unitedtaps = "UC5Q8e9-uutVZmiwAcEHKuzA"
srodalmenara = "UC9kc1DaOy2kSzXHZgpn9kfQ"
ShutupAndPlay = "UCAwuvzhah0KUw5QNihSkEwQ"
sanx2 = "UCJ_waKl9kjbhfXI3LJOfHvA"
DerickWattsAndTheSundayBlues = "UCmZhiZq7M7d73Kbey4yna_Q"
akirathedon = "UCsoiSpBvkr4Y-78Pj3recUw"
Musicoterapia = "UCsoSK8K4OpdMV1tqJFwO5QA"
daydreaming = "UCtbuGylbRXc42pIxWey19Dg"
RemixHolicRecords = "UCUW5GjwcXgbPcfk00t-GQZA"
EDMBot = "UCvmUdL2NHWlj1NRiNJPI-TQ"
TrioTravels = "UCdAPAmdnkdFsH5R2Hxevucg"
caosonnguyen = "UCwPeW9kFId5-VbQ2LQEjVhg"
TheAlmonteFilms = "UC4C_SF5koS4Q5om50b9NMTw"
timcast = "UCG749Dj4V2fKa143f8sE60Q"
SeekingTheTruth = "UCHrDpTVL9S0h91u9UCPgVbA"
davidpakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
mikenayna = "UCzk08fzh5c_BhjQa1w35wtA"
DoorMonster = "UC-to_wlckb-bFDtQfUZL3Kw"
barnacules = "UC1MwJy1R0nGQkXxRD9p-zTQ"
brightsunfilms = "UC5k3Kc0avyDJ2nG9Kxm9JmQ"
Onision = "UC5OxQNCgW88FDBxeZCnrBbg"
top10archive = "UCa03bf8gAS2EtffptV-_jfA"
thought = "UCb0yiUQhhLV_jpY3BayJaLA"
MothersBasement = "UCBs2Y3i14e1NWQxOGliatmg"
VlogsOfKnowledge = "UCDUPGR6uL5uz0_hiRAQjZMQ"
GorTheMovieGod = "UCHdTVw89QU6coU1MgN-9RHA"
iamalexoconnor = "UCKAQLEk1GGqnPtov9EW0huQ"
ADedits = "UCsX-zRuq3ovMsgFqEQLw2Bw"
SynthCool = "UCxGTHsD0pLSFlFI7M7jYmBQ"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
Sciencedocumentinhindi = "UC9SpfUF3rm-MGep5WE6FSCA"
NurdRage = "UCIgKGGJkt1MrNmhq3vRibYA"
MINDBLASTER = "UC_ZMqbRu44jK-EogjYyHz8g"
DaminousPurity = "UCdKRWvz50QoioZFgu6Nf9og"
ScammerRevolts = "UC0uJKUXiU5T41Fzawy5H6mw"
eevblog = "UC2DjFE7Xf11URZqWBigcVOQ"
Luke = "UC2eYFnH61tmytImy1mTYvhA"
AppGirl = "UC389S4_2Yt9cei1qNDwmBrA"
thecryptodad = "UC68x_TIzqCtF69fYl2_kl3w"
ChrisWereDigital = "UCAPR27YUyxmgwm3Wc2WSHLw"
alecaddd = "UCbmBY_XYZqCa2G0XmFA7ZWg"
EliTheComputerGuy = "UCD4EOyXKjfDUhCI6jlOZZYQ"
archetapp = "UCDIBBmkZIB2hjBsk1hUImdA"
PCPlaceNZ = "UCf5ZTSZAKbinY03jOwylfOg"
NaomiSexyCyborgWu = "UCh_ugKacslKhsGGdXP0cRRA"
NibiruWatcher = "UCi62JvN-lUn7hVL3ffofADA"
imineblocks = "UCjYHcWGAjUVqU49D2JOKD3w"
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
CooLoserTech = "UCl97rZ2Tc7KV9lktmmHNFDQ"
TechHD = "UCN3bPy04Jkp3ADRtyYvXomQ"
MiketheScrapper = "UCqtlJpkH_llXS_vuDExGVvw"
eevblog2 = "UCr-cm90DwFJC0W3f9jBs5jA"
thecreativeone = "UCTikFhzCiIXfOMS7D29dvYg"
weekendtricks = "UCYtAJXx0ymGPpCndn2Gt6-w"
GBGuns = "UC2VOURrALs1CwVmbGlXJOPQ"
Matsimus = "UCFWjEwhX6cSAKBQ28pufG3w"
TheLateBoyScout = "UCZjvj5MN3BMxPFfdEKIrvxQ"
BravoCinematografica = "UC2ruSXQoKMgr7JXzwO0H0KA"
PyroNation = "UC4ffy3n1hE7Z8q-2KVq_91Q"
dramatuber = "UC4Y8mImty3gFEG9grsR6T-w"
BarnabasNagy = "UC8TRZRK1sJfxKJ0tXMmGTow"
avery = "UCcfjIZLDCuSqkIlkH8i4DDg"
Tingledove = "UCfI7wtV6K64gVbzjH7DOA_Q"
crmjewelers = "UChpFWeF84jA5JeV3YyIo3pQ"
NerfNerd18 = "UCIgmaEJNqvH9bU9zGMapZGg"
SEIJIHITO = "UCNqUrLE6dI8fWw_u3HQkpXA"
anvithavlogs = "UCsP9pYat2DEBvnvF_iFGG_w"
YoelRekts = "UCZ_BcFyhIo6GdtTSrqXXepg"
TechFox = "UCIp-oTSdFO7BhAJpW2d5HMQ"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
Juggling: {"juggling", "circus arts", "malabares"},
JustJuggling: {"juggling", "circus arts", "malabares"},
JordanBPeterson: {"postmodernism", "psychology", "news"},
JustJuggling: {"juggling", "circus arts", "malabares"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
TechFox: {"technology", "reviews"},
misszizi: {"art", "pop culture"},
creationshub: {"art"},
AlicePandora: {"art"},
Akito0405: {"art"},
dashcambristol: {"automotive"},
adrianbonillap: {"automotive"},
SubhamVlogs: {"automotive"},
DisciplesofJesusChristJeremiahPayne: {"beliefs"},
ModernGalaxy: {"beliefs"},
socofilms: {"beliefs"},
buriedone: {"blockchain"},
bitcoin: {"blockchain"},
globalrashid: {"blockchain"},
gameofbitcoins: {"blockchain"},
CryptoInsightsBrasil: {"blockchain"},
Crypt0: {"blockchain"},
cryptomined: {"blockchain"},
altcoinbuzz: {"blockchain", "technology"},
crypto: {"blockchain"},
Crypto99: {"blockchain"},
TheCryptoLark: {"blockchain", "technology"},
NuggetsNews: {"blockchain", "learning"},
btckyle: {"blockchain"},
LouisThomas: {"blockchain"},
BitsBeTrippin: {"blockchain"},
cryptocrow: {"blockchain"},
KouSuccessLeeFX: {"economics", "learning"},
TheSchiffReport: {"economics"},
MRU: {"economics"},
Vidello: {"economics", "pop culture"},
kaccreative: {"food", "art"},
shecooks: {"food"},
vegsource: {"food"},
VeganGains: {"food"},
AwesomeKnowledge: {"funny"},
luckyloush: {"funny"},
MichaelLuzzi: {"funny", "pop culture"},
NakedApe: {"funny"},
ParaReact: {"funny"},
dullytv: {"funny"},
REDONKULAS: {"funny"},
RedactedTonight: {"funny", "news"},
TonyTornado: {"funny"},
hoodlumscrafty: {"gaming", "pop culture"},
Karmakut: {"gaming"},
Draxr: {"gaming", "pop culture"},
BigfryTV: {"gaming"},
CSGONews: {"gaming", "funny"},
ZedGaming: {"gaming"},
KamiVS: {"gaming"},
grabthegames: {"gaming"},
blueplays: {"gaming"},
ImNotBonkers: {"gaming", "funny"},
GunslingerMedia: {"gaming"},
LeagueVoices: {"gaming"},
SirPugger: {"gaming"},
gamesoup: {"gaming"},
ProHenis: {"gaming"},
Dota2Divine: {"gaming"},
CrazyFoxMovies: {"gaming", "pop culture", "technology"},
Larry: {"gaming"},
hottake: {"gaming", "funny"},
Breezy: {"gaming", "funny"},
retrorgb: {"gaming"},
Zer0Gamer: {"gaming"},
nickatnyte: {"gaming"},
DyllonStejGaming: {"gaming"},
bluedrake42: {"gaming"},
Rerez: {"gaming"},
Op: {"gaming"},
kidsgamesfun: {"gaming", "pop culture"},
GamesGlitches: {"gaming"},
Musikage: {"gaming", "pop culture"},
JeffyGaming: {"gaming", "funny"},
oniblackmage: {"gaming"},
nickatnyte2: {"gaming"},
KazeEmanuar: {"gaming"},
TheLinuxGamer: {"gaming", "technology", "linux"},
GaminGHD: {"gaming", "pop culture"},
BHGaming: {"gaming"},
Potato: {"gaming"},
MikeyTaylorGaming: {"gaming"},
Acituanbus: {"gaming"},
juggling: {"juggling", "circus art", "malabares"},
KhanAcademy: {"learning", "science"},
DON: {"learning", "pop culture"},
shogogarcia: {"learning"},
alphalifestyleacademy: {"learning"},
NileRed: {"learning", "science"},
veritasium: {"learning", "science"},
stevecronin: {"learning"},
jeranism: {"learning"},
MinutePhysics: {"learning", "science"},
_3Blue1Brown: {"learning"},
Itsrucka: {"music"},
unitedtaps: {"music"},
srodalmenara: {"music"},
ShutupAndPlay: {"music", "learning"},
sanx2: {"music"},
DerickWattsAndTheSundayBlues: {"music", "funny"},
akirathedon: {"music"},
Musicoterapia: {"music"},
daydreaming: {"music"},
RemixHolicRecords: {"music"},
EDMBot: {"music"},
TrioTravels: {"nature"},
caosonnguyen: {"nature"},
TheAlmonteFilms: {"news"},
timcast: {"news", "technology"},
SeekingTheTruth: {"news"},
davidpakman: {"news"},
mikenayna: {"news"},
DoorMonster: {"pop culture", "funny"},
barnacules: {"pop culture", "gaming"},
brightsunfilms: {"pop culture"},
Onision: {"pop culture", "funny"},
top10archive: {"pop culture"},
thought: {"pop culture", "learning"},
MothersBasement: {"pop culture", "gaming"},
VlogsOfKnowledge: {"pop culture", "gaming"},
GorTheMovieGod: {"pop culture"},
iamalexoconnor: {"pop culture"},
ADedits: {"pop culture"},
SynthCool: {"pop culture", "funny"},
JordanBPeterson: {"psychology", "postmodernism", "news"},
Sciencedocumentinhindi: {"science"},
NurdRage: {"science", "learning"},
MINDBLASTER: {"sports", "funny"},
DaminousPurity: {"sports", "gaming"},
ScammerRevolts: {"technology"},
eevblog: {"technology", "learning"},
Luke: {"technology", "funny"},
AppGirl: {"technology"},
thecryptodad: {"technology", "blockchain"},
ChrisWereDigital: {"technology"},
alecaddd: {"technology", "learning"},
EliTheComputerGuy: {"technology"},
archetapp: {"technology", "learning"},
PCPlaceNZ: {"technology"},
NaomiSexyCyborgWu: {"technology"},
NibiruWatcher: {"technology", "learning"},
imineblocks: {"technology", "blockchain"},
Lunduke: {"technology"},
CooLoserTech: {"technology"},
TechHD: {"technology", "learning"},
MiketheScrapper: {"technology"},
eevblog2: {"technology"},
thecreativeone: {"technology", "gaming"},
weekendtricks: {"technology"},
GBGuns: {"weapons"},
Matsimus: {"weapons", "gaming"},
TheLateBoyScout: {"weapons"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,
@@ -1,4 +1,4 @@
package tagsManager
package tags_manager
import (
"fmt"
+36 -19
View File
@@ -1,17 +1,19 @@
package thumbs
import (
"google.golang.org/api/youtube/v3"
"io"
"net/http"
"os"
"strings"
"github.com/lbryio/errors.go"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/prometheus/common/log"
log "github.com/sirupsen/logrus"
)
type thumbnailUploader struct {
@@ -25,13 +27,15 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
func (u *thumbnailUploader) downloadThumbnail() error {
_ = os.Mkdir(thumbnailPath, 0750)
_ = os.Mkdir(thumbnailPath, 0777)
img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
if err != nil {
return errors.Err(err)
}
defer img.Close()
if strings.HasPrefix(u.originalUrl, "//") {
u.originalUrl = "https:" + u.originalUrl
}
resp, err := http.Get(u.originalUrl)
if err != nil {
return errors.Err(err)
@@ -61,11 +65,14 @@ func (u *thumbnailUploader) uploadThumbnail() error {
uploader := s3manager.NewUploader(s3Session)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
CacheControl: aws.String("public, max-age=2592000"),
})
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)
}
@@ -93,18 +100,28 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
return "", err
}
ownS3Config := s3Config.Copy(&aws.Config{Endpoint: aws.String("s3.lbry.tech")})
tu2 := thumbnailUploader{
originalUrl: url,
name: name,
s3Config: *ownS3Config,
}
//own S3
err = tu2.uploadThumbnail()
if err != nil {
return "", err
}
return tu.mirroredUrl, nil
}
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
if thumbnails.Maxres != nil {
return thumbnails.Maxres
} else if thumbnails.High != nil {
return thumbnails.High
} else if thumbnails.Medium != nil {
return thumbnails.Medium
} else if thumbnails.Standard != nil {
return thumbnails.Standard
func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
var bestWidth ytdl.Thumbnail
for _, thumbnail := range thumbnails {
if bestWidth.Width < thumbnail.Width {
bestWidth = thumbnail
}
}
return thumbnails.Default
return &bestWidth
}
+114
View File
@@ -0,0 +1,114 @@
package timing
import (
"sync"
"sync/atomic"
"time"
"github.com/lbryio/ytsync/v5/metrics"
"github.com/sirupsen/logrus"
)
type Timing struct {
component string
milliseconds int64
min int64
max int64
invocations int32
}
var timings *sync.Map
func TimedComponent(component string) *Timing {
if timings == nil {
timings = &sync.Map{}
}
stored, _ := timings.LoadOrStore(component, &Timing{
component: component,
milliseconds: 0,
min: int64(99999999),
})
t, _ := stored.(*Timing)
return t
}
func ClearTimings() {
if timings == nil {
return
}
timings.Range(func(key interface{}, value interface{}) bool {
timings.Delete(key)
return true
})
}
func Report() {
var totalTime time.Duration
timings.Range(func(key interface{}, value interface{}) bool {
totalTime += value.(*Timing).Get()
return true
})
timings.Range(func(key interface{}, value interface{}) bool {
component := key
componentRuntime := value.(*Timing).Get().String()
percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
invocations := value.(*Timing).Invocations()
avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
minRuntime := value.(*Timing).Min().String()
maxRuntime := value.(*Timing).Max().String()
logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
component,
componentRuntime,
percentTime,
invocations,
avgTime,
minRuntime,
maxRuntime,
)
return true
})
}
func (t *Timing) Add(d time.Duration) {
metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
atomic.AddInt64(&t.milliseconds, d.Milliseconds())
for {
oldMin := atomic.LoadInt64(&t.min)
if d.Milliseconds() < oldMin {
if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
break
}
} else {
break
}
}
for {
oldMax := atomic.LoadInt64(&t.max)
if d.Milliseconds() > oldMax {
if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
break
}
} else {
break
}
}
atomic.AddInt32(&t.invocations, 1)
}
func (t *Timing) Get() time.Duration {
ms := atomic.LoadInt64(&t.milliseconds)
return time.Duration(ms) * time.Millisecond
}
func (t *Timing) Invocations() int32 {
return atomic.LoadInt32(&t.invocations)
}
func (t *Timing) Min() time.Duration {
ms := atomic.LoadInt64(&t.min)
return time.Duration(ms) * time.Millisecond
}
func (t *Timing) Max() time.Duration {
ms := atomic.LoadInt64(&t.max)
return time.Duration(ms) * time.Millisecond
}
+34
View File
@@ -0,0 +1,34 @@
package util
import (
"fmt"
"github.com/lbryio/lbry.go/v2/extras/util"
log "github.com/sirupsen/logrus"
)
// SendErrorToSlack Sends an error message to the default channel and to the process log.
func SendErrorToSlack(format string, a ...interface{}) {
message := format
if len(a) > 0 {
message = fmt.Sprintf(format, a...)
}
log.Errorln(message)
err := util.SendToSlack(":sos: " + message)
if err != nil {
log.Errorln(err)
}
}
// SendInfoToSlack Sends an info message to the default channel and to the process log.
func SendInfoToSlack(format string, a ...interface{}) {
message := format
if len(a) > 0 {
message = fmt.Sprintf(format, a...)
}
log.Infoln(message)
err := util.SendToSlack(":information_source: " + message)
if err != nil {
log.Errorln(err)
}
}
+421
View File
@@ -0,0 +1,421 @@
package util
import (
"context"
"os"
"os/exec"
"os/user"
"path/filepath"
"strconv"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/ytsync/v5/timing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/mitchellh/go-ps"
log "github.com/sirupsen/logrus"
)
func GetBlobsDir() string {
blobsDir := os.Getenv("BLOBS_DIRECTORY")
if blobsDir == "" {
usr, err := user.Current()
if err != nil {
log.Error(err.Error())
return ""
}
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
}
return blobsDir
}
func IsBlobReflectionOff() bool {
return os.Getenv("REFLECT_BLOBS") == "false"
}
func GetLBRYNetDir() string {
lbrynetDir := os.Getenv("LBRYNET_DIR")
if lbrynetDir == "" {
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return ""
}
return usr.HomeDir + "/.lbrynet/"
}
return lbrynetDir
}
func GetLbryumDir() string {
lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir == "" {
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return ""
}
return usr.HomeDir + "/.lbryum/"
}
return lbryumDir + "/"
}
const ALL = true
const ONLINE = false
func GetLBRYNetContainer(all bool) (*types.Container, error) {
return getDockerContainer("lbrynet", all)
}
func getDockerContainer(name string, all bool) (*types.Container, error) {
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
filters := filters.NewArgs()
filters.Add("name", name)
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters})
if err != nil {
panic(err)
}
if len(containers) == 0 {
return nil, nil
}
if len(containers) > 1 {
return nil, errors.Err("more than one %s container found", name)
}
return &containers[0], nil
}
func IsUsingDocker() bool {
useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER"))
if err != nil {
return false
}
return useDocker
}
func IsRegTest() bool {
usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST"))
if err != nil {
return false
}
return usesRegtest
}
func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
chainName := os.Getenv("CHAINNAME")
chainParams, ok := lbrycrd.ChainParamsMap[chainName]
if !ok {
chainParams = lbrycrd.MainNetParams
}
var lbrycrdd *lbrycrd.Client
var err error
if lbrycrdString == "" {
lbrycrdd, err = lbrycrd.NewWithDefaultURL(&chainParams)
if err != nil {
return nil, err
}
} else {
lbrycrdd, err = lbrycrd.New(lbrycrdString, &chainParams)
if err != nil {
return nil, err
}
}
return lbrycrdd, nil
}
func ShouldCleanOnStartup() bool {
shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP"))
if err != nil {
return false
}
return shouldClean
}
func IsLbrynetRunning() (bool, error) {
if IsUsingDocker() {
container, err := GetLBRYNetContainer(ONLINE)
if err != nil {
return false, err
}
return container != nil, nil
}
processes, err := ps.Processes()
if err != nil {
return true, errors.Err(err)
}
var daemonProcessId = -1
for _, p := range processes {
if p.Executable() == "lbrynet" {
daemonProcessId = p.Pid()
break
}
}
running := daemonProcessId != -1
return running, nil
}
func CleanForStartup() error {
if !IsRegTest() {
return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
}
running, err := IsLbrynetRunning()
if err != nil {
return err
}
if running {
err := StopDaemon()
if err != nil {
return err
}
}
err = CleanupLbrynet()
if err != nil {
return errors.Err(err)
}
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
height, err := lbrycrd.GetBlockCount()
if err != nil {
return errors.Err(err)
}
const minBlocksForUTXO = 200
if height < minBlocksForUTXO {
//Start reg test with some credits
txs, err := lbrycrd.Generate(uint32(minBlocksForUTXO) - uint32(height))
if err != nil {
return errors.Err(err)
}
log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
}
defaultWalletDir := GetDefaultWalletPath()
_, err = os.Stat(defaultWalletDir)
if os.IsNotExist(err) {
return nil
}
return errors.Err(os.Remove(defaultWalletDir))
}
func CleanupLbrynet() error {
//make sure lbrynet is off
running, err := IsLbrynetRunning()
if err != nil {
return err
}
if running {
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
}
lbrynetDir := GetLBRYNetDir()
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
if err != nil {
return errors.Err(err)
}
for _, f := range files {
err = os.Remove(f)
if err != nil {
return errors.Err(err)
}
}
blobsDir := GetBlobsDir()
err = os.RemoveAll(blobsDir)
if err != nil {
return errors.Err(err)
}
err = os.Mkdir(blobsDir, 0777)
if err != nil {
return errors.Err(err)
}
lbryumDir := GetLbryumDir()
ledger := "lbc_mainnet"
if IsRegTest() {
ledger = "lbc_regtest"
}
lbryumDir = lbryumDir + ledger
files, err = filepath.Glob(lbryumDir + "/blockchain.db*")
if err != nil {
return errors.Err(err)
}
for _, f := range files {
err = os.Remove(f)
if err != nil {
return errors.Err(err)
}
}
return nil
}
var metadataDirInitialized = false
func GetVideoMetadataDir() string {
dir := "./videos_metadata"
if !metadataDirInitialized {
metadataDirInitialized = true
_ = os.MkdirAll(dir, 0755)
}
return dir
}
func CleanupMetadata() error {
dir := GetVideoMetadataDir()
err := os.RemoveAll(dir)
if err != nil {
return errors.Err(err)
}
metadataDirInitialized = false
return nil
}
func SleepUntilQuotaReset() {
PST, _ := time.LoadLocation("America/Los_Angeles")
t := time.Now().In(PST)
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
d := n.Sub(t)
if d < 0 {
n = n.Add(24 * time.Hour)
d = n.Sub(t)
}
log.Infof("gotta sleep %s until the quota resets", d.String())
time.Sleep(d)
}
func StartDaemon() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("startDaemon").Add(time.Since(start))
}(start)
if IsUsingDocker() {
return startDaemonViaDocker()
}
return startDaemonViaSystemd()
}
func StopDaemon() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("stopDaemon").Add(time.Since(start))
}(start)
if IsUsingDocker() {
return stopDaemonViaDocker()
}
return stopDaemonViaSystemd()
}
func startDaemonViaDocker() error {
container, err := GetLBRYNetContainer(true)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaDocker() error {
container, err := GetLBRYNetContainer(ONLINE)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStop(context.Background(), container.ID, nil)
if err != nil {
return errors.Err(err)
}
return nil
}
func startDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
if err != nil {
return errors.Err(err)
}
return nil
}
func GetDefaultWalletPath() string {
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
if IsRegTest() {
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
}
walletPath := os.Getenv("LBRYUM_DIR")
if walletPath != "" {
defaultWalletDir = walletPath + "/wallets/default_wallet"
}
return defaultWalletDir
}
func GetBlockchainDBPath() string {
lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir == "" {
if IsRegTest() {
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
} else {
lbryumDir = os.Getenv("HOME") + "/.lbryum"
}
}
defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
if IsRegTest() {
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
}
return defaultDB
}
func GetBlockchainDirectoryName() string {
ledger := "lbc_mainnet"
if IsRegTest() {
ledger = "lbc_regtest"
}
return ledger
}
func DirSize(path string) (int64, error) {
var size int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
size += info.Size()
}
return err
})
return size, err
}
+393
View File
@@ -0,0 +1,393 @@
package ytapi
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/ytsync/v5/shared"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/vbauerster/mpb/v7"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/ytsync/v5/downloader"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/sources"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/aws/aws-sdk-go/aws"
log "github.com/sirupsen/logrus"
)
type Video interface {
Size() *int64
ID() string
IDAndNum() string
PlaylistPosition() int
PublishedAt() time.Time
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex, *sync.WaitGroup, *mpb.Progress) (*sources.SyncSummary, error)
}
type byPublishedAt []Video
func (a byPublishedAt) Len() int { return len(a) }
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
type VideoParams struct {
VideoDir string
S3Config aws.Config
Stopper *stop.Group
IPPool *ip_manager.IPPool
}
var mostRecentlyFailedChannel string // TODO: fix this hack!
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
var videos []Video
if quickSync && maxVideos > 50 {
maxVideos = 50
}
allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
if err != nil {
return nil, errors.Err(err)
}
videoIDs := make([]string, 0, len(allVideos))
for _, video := range allVideos {
sv, ok := syncedVideos[video]
if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
continue
}
videoIDs = append(videoIDs, video)
}
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
playlistMap := make(map[string]int64)
for i, videoID := range videoIDs {
playlistMap[videoID] = int64(i)
}
//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
if lastUploadedVideo != "" {
_, ok := playlistMap[lastUploadedVideo]
if !ok {
playlistMap[lastUploadedVideo] = 0
videoIDs = append(videoIDs, lastUploadedVideo)
}
}
if len(videoIDs) < 1 {
if channelID == mostRecentlyFailedChannel {
return nil, errors.Err("playlist items not found")
}
mostRecentlyFailedChannel = channelID
}
vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
if err != nil {
return nil, err
}
for _, item := range vids {
positionInList := playlistMap[item.ID]
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
if err != nil {
return nil, errors.Err(err)
}
videos = append(videos, videoToAdd)
}
for k, v := range syncedVideos {
if !v.Published {
continue
}
if _, ok := playlistMap[k]; !ok {
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
}
}
sort.Sort(byPublishedAt(videos))
return videos, nil
}
// CountVideosInChannel is unused for now... keeping it here just in case
func CountVideosInChannel(channelID string) (int, error) {
url := "https://socialblade.com/youtube/channel/" + channelID
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("User-Agent", downloader.ChromeUA)
req.Header.Add("Accept", "*/*")
req.Header.Add("Host", "socialblade.com")
res, err := http.DefaultClient.Do(req)
if err != nil {
return 0, errors.Err(err)
}
defer res.Body.Close()
var line string
scanner := bufio.NewScanner(res.Body)
for scanner.Scan() {
if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
line = scanner.Text()
break
}
}
if err := scanner.Err(); err != nil {
return 0, err
}
if line == "" {
return 0, errors.Err("upload count line not found")
}
matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
if len(matches) != 2 {
return 0, errors.Err("upload count not found with regex")
}
num, err := strconv.Atoi(matches[1])
if err != nil {
return 0, errors.Err(err)
}
return num, nil
}
func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
url := "https://www.youtube.com/channel/" + channelID + "/about"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("User-Agent", downloader.ChromeUA)
req.Header.Add("Accept", "*/*")
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, errors.Err(err)
}
pageBody := string(body)
dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
if dataStartIndex == 25 {
dataStartIndex = strings.Index(pageBody, "var ytInitialData = ") + 20
}
dataEndIndex := strings.Index(pageBody, "]}}};") + 4
if dataEndIndex < dataStartIndex {
return nil, errors.Err("start index is lower than end index. cannot extract channel info!")
}
data := pageBody[dataStartIndex:dataEndIndex]
var decodedResponse YoutubeStatsResponse
err = json.Unmarshal([]byte(data), &decodedResponse)
if err != nil {
return nil, errors.Err(err)
}
return &decodedResponse, nil
}
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
var videos []*ytdl.YtdlVideo
for _, videoID := range videoIDs {
if len(videoID) < 5 {
continue
}
select {
case <-stopChan:
return videos, errors.Err("interrupted by user")
default:
}
//ip, err := ipPool.GetIP(videoID)
//if err != nil {
// return nil, err
//}
//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
state, err := config.VideoState(videoID)
if err != nil {
return nil, errors.Err(err)
}
if state == "published" {
continue
}
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
if err != nil {
errSDK := config.MarkVideoStatus(shared.VideoStatus{
ChannelID: channelID,
VideoID: videoID,
Status: "failed",
FailureReason: err.Error(),
})
logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
if errSDK != nil {
return nil, errors.Err(errSDK)
}
} else {
videos = append(videos, video)
}
}
return videos, nil
}
type YoutubeStatsResponse struct {
Contents struct {
TwoColumnBrowseResultsRenderer struct {
Tabs []struct {
TabRenderer struct {
Title string `json:"title"`
Selected bool `json:"selected"`
Content struct {
SectionListRenderer struct {
Contents []struct {
ItemSectionRenderer struct {
Contents []struct {
ChannelAboutFullMetadataRenderer struct {
Description struct {
SimpleText string `json:"simpleText"`
} `json:"description"`
ViewCountText struct {
SimpleText string `json:"simpleText"`
} `json:"viewCountText"`
JoinedDateText struct {
Runs []struct {
Text string `json:"text"`
} `json:"runs"`
} `json:"joinedDateText"`
CanonicalChannelURL string `json:"canonicalChannelUrl"`
BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"`
Title struct {
SimpleText string `json:"simpleText"`
} `json:"title"`
Avatar struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"avatar"`
ShowDescription bool `json:"showDescription"`
DescriptionLabel struct {
Runs []struct {
Text string `json:"text"`
} `json:"runs"`
} `json:"descriptionLabel"`
DetailsLabel struct {
Runs []struct {
Text string `json:"text"`
} `json:"runs"`
} `json:"detailsLabel"`
ChannelID string `json:"channelId"`
} `json:"channelAboutFullMetadataRenderer"`
} `json:"contents"`
} `json:"itemSectionRenderer"`
} `json:"contents"`
} `json:"sectionListRenderer"`
} `json:"content"`
} `json:"tabRenderer"`
} `json:"tabs"`
} `json:"twoColumnBrowseResultsRenderer"`
} `json:"contents"`
Header struct {
C4TabbedHeaderRenderer struct {
ChannelID string `json:"channelId"`
Title string `json:"title"`
Avatar struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"avatar"`
Banner struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"banner"`
VisitTracking struct {
RemarketingPing string `json:"remarketingPing"`
} `json:"visitTracking"`
SubscriberCountText struct {
SimpleText string `json:"simpleText"`
} `json:"subscriberCountText"`
} `json:"c4TabbedHeaderRenderer"`
} `json:"header"`
Metadata struct {
ChannelMetadataRenderer struct {
Title string `json:"title"`
Description string `json:"description"`
RssURL string `json:"rssUrl"`
ChannelConversionURL string `json:"channelConversionUrl"`
ExternalID string `json:"externalId"`
Keywords string `json:"keywords"`
OwnerUrls []string `json:"ownerUrls"`
Avatar struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"avatar"`
ChannelURL string `json:"channelUrl"`
IsFamilySafe bool `json:"isFamilySafe"`
VanityChannelURL string `json:"vanityChannelUrl"`
} `json:"channelMetadataRenderer"`
} `json:"metadata"`
Topbar struct {
DesktopTopbarRenderer struct {
CountryCode string `json:"countryCode"`
} `json:"desktopTopbarRenderer"`
} `json:"topbar"`
Microformat struct {
MicroformatDataRenderer struct {
URLCanonical string `json:"urlCanonical"`
Title string `json:"title"`
Description string `json:"description"`
Thumbnail struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"thumbnail"`
SiteName string `json:"siteName"`
AppName string `json:"appName"`
AndroidPackage string `json:"androidPackage"`
IosAppStoreID string `json:"iosAppStoreId"`
IosAppArguments string `json:"iosAppArguments"`
OgType string `json:"ogType"`
URLApplinksWeb string `json:"urlApplinksWeb"`
URLApplinksIos string `json:"urlApplinksIos"`
URLApplinksAndroid string `json:"urlApplinksAndroid"`
URLTwitterIos string `json:"urlTwitterIos"`
URLTwitterAndroid string `json:"urlTwitterAndroid"`
TwitterCardType string `json:"twitterCardType"`
TwitterSiteHandle string `json:"twitterSiteHandle"`
SchemaDotOrgType string `json:"schemaDotOrgType"`
Noindex bool `json:"noindex"`
Unlisted bool `json:"unlisted"`
FamilySafe bool `json:"familySafe"`
Tags []string `json:"tags"`
} `json:"microformatDataRenderer"`
} `json:"microformat"`
}
+13
View File
@@ -0,0 +1,13 @@
package ytapi
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestChannelInfo(t *testing.T) {
info, err := ChannelInfo("UCNQfQvFMPnInwsU_iGYArJQ")
assert.NoError(t, err)
assert.NotNil(t, info)
}