Compare commits

...
20 Commits
Author SHA1 Message Date
Niko Storni 7f48220a80 remove dist folder... 2019-06-26 03:48:44 +02:00
Niko Storni 9e49110295 fix hash 2019-06-26 03:37:52 +02:00
Niko Storni 4506bdb638 fix bug in utxo counting 2019-06-25 21:12:15 -04:00
Niko Storni a8551bc9ee fix UTXO waiting logic 2019-06-25 20:45:27 -04:00
Niko Storni b5c9806dd0 improve space management 2019-06-24 20:43:50 -04:00
Niko Storni 8c349cf7ac fix bug in queues 2019-06-24 04:14:27 -04:00
Niko Storni f3f40c41ef fix refill balance 2019-06-22 06:08:11 -04:00
Niko Storni 05b65c4cb5 add noretry failure 2019-06-21 16:41:46 +02:00
Niko Storni 69ad13848b fix utxo count
fix one bug
add content type to uploaded thumbs
2019-06-20 21:45:17 +02:00
Niko Storni e496ad670a limit tag length 2019-06-20 17:43:46 +02:00
Niko 9d8eb32f9f Merge pull request #28 from lbryio/metadata_upgrade
Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata
2019-06-13 20:14:13 +02:00
Niko Storni 6668840b11 fix description not containing youtube link
add workaround to SDK bug not spending the right UTXOs
fix bug in assignments
2019-06-13 19:33:58 +02:00
Niko Storni e893f68bc1 fix queue pulling
fix address reuse
2019-06-12 22:42:42 +02:00
Niko Storni 8da631eac3 add a limit to UTXOs generation
fix broadcastFee
fix spelling
2019-06-12 05:33:13 +02:00
Niko Storni 1729af0ae5 fix UTXOs management while publishing
fix account_set
improve logging
2019-06-12 05:25:01 +02:00
Niko Storni dce71d2451 add adjustments to video tracking
reduce minimum refill amount (min account balance supersedes)
revert account_fund change as --everything isn't supported with outputs
fix bugs
2019-06-12 04:44:56 +02:00
Niko Storni f4e75cf221 rewrite account funding algorithm
lock publishing when UTXOs management is in progress
spend everything when funding UTXOs
update lbry.go
2019-06-12 03:17:59 +02:00
Niko Storni ae3dad944d add blocking support for publishing and upgrading
add channel wide mappings
2019-06-11 19:18:39 +02:00
Niko Storni f7f0231c80 add VideoUpgradeFailed staus 2019-06-10 21:59:42 +02:00
Niko Storni 0c42b96332 remove unused script
update readme
2019-06-10 21:43:10 +02:00
12 changed files with 584 additions and 299 deletions

No files matched your search

+3 -1
View File
@@ -55,7 +55,7 @@ Usage:
Flags:
--after int Specify from when to pull jobs [Unix time](Default: 0)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1582834707)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
--channelID string If specified, only this channel will be synced.
--concurrent-jobs int how many jobs to process concurrently (default 1)
-h, --help help for ytsync
@@ -63,12 +63,14 @@ Flags:
--max-length float Maximum video length to process (in hours) (default 2)
--max-size int Maximum video size to process (in MB) (default 2048)
--max-tries int Number of times to try a publish that fails (default 3)
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
--run-once Whether the process should be stopped after one cycle or not
--skip-space-check Do not perform free space check on startup
--status string Specify which queue to pull from. Overrides --update
--stop-on-error If a publish fails, stop all publishing and exit
--takeover-existing-channel If channel exists and we don't own it, take over the channel
--update Update previously synced channels instead of syncing new ones
--upgrade-metadata Upgrade videos if they're on the old metadata version
--videos-limit int how many videos to process per channel (default 1000)
```
+1 -1
View File
@@ -15,7 +15,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.9
github.com/lbryio/lbry.go v1.0.15
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
+2 -23
View File
@@ -9,7 +9,6 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -18,7 +17,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -31,9 +29,7 @@ github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
@@ -60,7 +56,6 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -70,9 +65,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -84,14 +77,12 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@@ -102,22 +93,17 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -129,17 +115,15 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v1.0.9 h1:SAqopNiISazYuF38quA3WEfqOCzFtUed6QfCkAYQoJo=
github.com/lbryio/lbry.go v1.0.9/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
@@ -157,11 +141,9 @@ github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457/go.mod h1:Z/EmaCtMz
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -197,7 +179,6 @@ github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -233,7 +214,6 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
@@ -244,7 +224,6 @@ golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+12 -7
View File
@@ -91,9 +91,10 @@ const (
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUnpublished = "unpublished"
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
)
func (s *SyncManager) Start() error {
@@ -141,6 +142,7 @@ func (s *SyncManager) Start() error {
shouldInterruptLoop = true
} else {
var queuesToSync []string
//TODO: implement scrambling to avoid starvation of queues
if s.syncStatus != "" {
queuesToSync = append(queuesToSync, s.syncStatus)
} else if s.syncUpdate {
@@ -153,7 +155,9 @@ func (s *SyncManager) Start() error {
if err != nil {
return err
}
for _, c := range channels {
log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
if len(channels) > 0 {
c := channels[0]
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: c.ChannelId,
@@ -173,6 +177,7 @@ func (s *SyncManager) Start() error {
namer: namer.NewNamer(),
Fee: c.Fee,
})
continue
}
}
}
@@ -180,9 +185,9 @@ func (s *SyncManager) Start() error {
log.Infoln("No channels to sync. Pausing 5 minutes!")
time.Sleep(5 * time.Minute)
}
for i, sync := range syncs {
for _, sync := range syncs {
shouldNotCount := false
SendInfoToSlack("Syncing %s (%s) to LBRY! (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
err := sync.FullCycle()
if err != nil {
fatalErrors := []string{
@@ -202,7 +207,7 @@ func (s *SyncManager) Start() error {
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
}
}
SendInfoToSlack("Syncing %s (%s) reached an end. (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
if !shouldNotCount {
syncCount++
}
+56 -49
View File
@@ -22,8 +22,6 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 4
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
if err != nil {
@@ -35,7 +33,8 @@ func (s *Sync) enableAddressReuse() error {
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: 1000,
ChangeMaxUses: util.PtrToInt(1000),
ReceivingMaxUses: util.PtrToInt(100),
})
if err != nil {
return errors.Err(err)
@@ -44,7 +43,7 @@ func (s *Sync) enableAddressReuse() error {
return nil
}
func (s *Sync) walletSetup() error {
//prevent unnecessary concurrent execution
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
s.walletMux.Lock()
defer s.walletMux.Unlock()
err := s.ensureChannelOwnership()
@@ -68,51 +67,51 @@ func (s *Sync) walletSetup() error {
if err != nil {
return err
}
numOnSource := int(n)
videosOnYoutube := int(n)
log.Debugf("Source channel has %d videos", numOnSource)
if numOnSource == 0 {
log.Debugf("Source channel has %d videos", videosOnYoutube)
if videosOnYoutube == 0 {
return nil
}
s.syncedVideosMux.RLock()
numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones...
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d videos", numPublished)
if numOnSource-numPublished > s.Manager.videosLimit {
numOnSource = s.Manager.videosLimit
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if s.Manager.upgradeMetadata {
videosToUpgrade := 0
for _, v := range s.syncedVideos {
if v.Published && v.MetadataVersion < 2 {
videosToUpgrade++
publishedCount := 0
notUpgradedCount := 0
failedCount := 0
for _, sv := range s.syncedVideos {
if sv.Published {
publishedCount++
if sv.MetadataVersion < 2 {
notUpgradedCount++
}
} else {
failedCount++
}
minBalance += float64(videosToUpgrade) * 0.001
}
if numPublished > numOnSource && balance < minimumRefillAmount {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = minimumRefillAmount
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
if videosOnYoutube > s.Manager.videosLimit {
videosOnYoutube = s.Manager.videosLimit
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
if s.Manager.upgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
}
refillAmount := 0.0
if balance < requiredBalance || balance < minimumAccountBalance {
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
}
amountToAdd := minBalance - balance
if s.Refill > 0 {
if amountToAdd < 0 {
amountToAdd = float64(s.Refill)
} else {
amountToAdd += float64(s.Refill)
}
refillAmount += float64(s.Refill)
}
if amountToAdd > 0 {
if amountToAdd < minimumRefillAmount {
amountToAdd = minimumRefillAmount
}
err := s.addCredits(amountToAdd)
if refillAmount > 0 {
err := s.addCredits(refillAmount)
if err != nil {
return errors.Err(err)
}
@@ -167,14 +166,19 @@ func (s *Sync) ensureEnoughUTXOs() error {
target := 40
slack := int(float32(0.1) * float32(target))
count := 0
confirmedCount := 0
for _, utxo := range *utxolist {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
if !utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
confirmedCount++
}
count++
}
}
log.Infof("utxo count: %d", count)
log.Infof("utxo count: %d (%d confirmed)", count, confirmedCount)
UTXOWaitThreshold := 16
if count < target-slack {
balance, err := s.daemon.AccountBalance(&defaultAccount)
if err != nil {
@@ -187,24 +191,27 @@ func (s *Sync) ensureEnoughUTXOs() error {
if err != nil {
return errors.Err(err)
}
broadcastFee := 0.01
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
desiredUTXOCount := uint64(math.Floor((balanceAmount - broadcastFee) / 0.1))
maxUTXOs := uint64(500)
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
if desiredUTXOCount > maxUTXOs {
desiredUTXOCount = maxUTXOs
}
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, desiredUTXOCount)
broadcastFee := 0.1
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
if err != nil {
return err
} else if prefillTx == nil {
return errors.Err("no response")
}
err = s.waitForNewBlock()
if err != nil {
return err
if confirmedCount < UTXOWaitThreshold {
err = s.waitForNewBlock()
if err != nil {
return err
}
}
} else if !allUTXOsConfirmed(utxolist) {
} else if confirmedCount < UTXOWaitThreshold {
log.Println("Waiting for previous txns to confirm")
err := s.waitForNewBlock()
if err != nil {
@@ -399,7 +406,7 @@ func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
}
for _, utxo := range *utxolist {
if utxo.Height == 0 {
if utxo.Confirmations <= 0 {
return false
}
}
+62 -34
View File
@@ -37,9 +37,12 @@ import (
)
const (
channelClaimAmount = 0.01
publishAmount = 0.01
maxReasonLength = 500
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.1
minimumAccountBalance = 4.0
minimumRefillAmount = 1
publishAmount = 0.01
maxReasonLength = 500
)
type video interface {
@@ -48,7 +51,7 @@ type video interface {
IDAndNum() string
PlaylistPosition() int
PublishedAt() time.Time
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool) (*sources.SyncSummary, error)
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
}
// sorting videos
@@ -83,17 +86,21 @@ type Sync struct {
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.Mutex
walletMux *sync.RWMutex
queue chan video
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
s.syncedVideosMux.Lock()
defer s.syncedVideosMux.Unlock()
s.syncedVideos[videoID] = sdk.SyncedVideo{
VideoID: videoID,
Published: published,
FailureReason: failureReason,
VideoID: videoID,
Published: published,
FailureReason: failureReason,
ClaimID: claimID,
ClaimName: claimName,
MetadataVersion: metadataVersion,
Size: size,
}
}
@@ -254,7 +261,7 @@ func (s *Sync) FullCycle() (e error) {
s.setExceptions()
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.Mutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stop.New()
s.queue = make(chan video)
interruptChan := make(chan os.Signal, 1)
@@ -288,6 +295,7 @@ func (s *Sync) FullCycle() (e error) {
return errors.Wrap(err, 0)
}
defer deleteSyncFolder(s.videoDirectory)
log.Printf("Starting daemon")
err = startDaemonViaSystemd()
if err != nil {
@@ -316,6 +324,16 @@ func (s *Sync) FullCycle() (e error) {
return nil
}
func deleteSyncFolder(videoDirectory string) {
if !strings.Contains(videoDirectory, "/tmp/ytsync") {
_ = util.SendToSlack(errors.Err("Trying to delete an unexpected directory: %s", videoDirectory).Error())
}
err := os.RemoveAll(videoDirectory)
if err != nil {
_ = util.SendToSlack(err.Error())
}
}
func (s *Sync) setChannelTerminationStatus(e *error) {
if *e != nil {
//conditions for which a channel shouldn't be marked as failed
@@ -473,7 +491,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
}
if claimIDDiffers {
if claimNameDiffers {
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
}
if claimMarkedUnpublished {
@@ -638,7 +656,7 @@ func (s *Sync) startWorker(workerNum int) {
err := s.processVideo(v)
if err != nil {
logMsg := "error processing video: " + err.Error()
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
log.Errorln(logMsg)
fatalErrors := []string{
":5279: read: connection reset by peer",
@@ -668,25 +686,29 @@ func (s *Sync) startWorker(workerNum int) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"Invalid status code: 403",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
log.Println("waiting for a block before retrying")
err := s.waitForNewBlock()
if err != nil {
s.grp.Stop()
SendErrorToSlack("something went wrong while waiting for a block: %v", err)
break
}
} else if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
} else if tryCount < s.MaxTries {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
log.Println("waiting for a block before retrying")
err = s.waitForNewBlock()
if err != nil {
s.grp.Stop()
SendErrorToSlack("something went wrong while waiting for a block: %v", err)
break
}
} else if util.SubstringInSlice(err.Error(), []string{
if util.SubstringInSlice(err.Error(), []string{
"Not enough funds to cover this transaction",
"failed: Not enough funds",
"Error in daemon: Insufficient funds, please deposit additional LBC"}) {
log.Println("refilling addresses before retrying")
err = s.walletSetup()
"Error in daemon: Insufficient funds, please deposit additional LBC",
// "txn-mempool-conflict", //TODO: uncomment the two lines when the SDK will start spending confirmed UTXOs before failing
//"too-long-mempool-chain",
}) {
log.Println("checking funds and UTXOs before retrying...")
err := s.walletSetup()
if err != nil {
s.grp.Stop()
SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
@@ -703,17 +725,24 @@ func (s *Sync) startWorker(workerNum int) {
existingClaim, ok := s.syncedVideos[v.ID()]
existingClaimID := ""
existingClaimName := ""
existingClaimSize := v.Size()
existingClaimSize := int64(0)
if v.Size() != nil {
existingClaimSize = *v.Size()
}
if ok {
existingClaimID = existingClaim.ClaimID
existingClaimName = existingClaim.ClaimName
if existingClaim.Size > 0 {
existingClaimSize = &existingClaim.Size
existingClaimSize = existingClaim.Size
}
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 0)
videoStatus := VideoStatusFailed
if strings.Contains(err.Error(), "upgrade failed") {
videoStatus = VideoStatusUpgradeFailed
} else {
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@@ -896,18 +925,17 @@ func (s *Sync) processVideo(v video) (err error) {
Fee: s.Fee,
}
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade)
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
if err != nil {
return err
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName)
return nil
}
+5 -4
View File
@@ -164,8 +164,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusPublished = "published"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusFailed = "failed"
)
func (a *APIConfig) DeleteVideos(videos []string) error {
@@ -207,9 +208,9 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
"status": {status},
"auth_token": {a.ApiToken},
}
if status == VideoStatusPublished {
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
if claimID == "" || claimName == "" {
return errors.Err("claimID or claimName missing")
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
+4 -1
View File
@@ -2,6 +2,7 @@ package sources
import (
"strings"
"sync"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/ytsync/namer"
@@ -12,7 +13,9 @@ type SyncSummary struct {
ClaimName string
}
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer) (*SyncSummary, error) {
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer, walletLock *sync.RWMutex) (*SyncSummary, error) {
walletLock.RLock()
defer walletLock.RUnlock()
for {
name := namer.GetNextName(title)
response, err := daemon.StreamCreate(name, filename, amount, options)
+20 -8
View File
@@ -10,6 +10,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
@@ -46,6 +47,7 @@ type YoutubeVideo struct {
thumbnailURL string
lbryChannelID string
mocked bool
walletLock *sync.RWMutex
}
const reflectorURL = "http://blobs.lbry.io/"
@@ -160,33 +162,40 @@ func (v *YoutubeVideo) getFullPath() string {
func (v *YoutubeVideo) getAbbrevDescription() string {
maxLines := 10
description := strings.TrimSpace(v.description)
if strings.Count(description, "\n") < maxLines {
return description
}
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
if v.lbryChannelID == khanAcademyClaimID {
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
}
if strings.Count(description, "\n") < maxLines {
return description + "\n..." + additionalDescription
}
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4")
"mp4",
"https://www.youtube.com/watch?v="+v.ID())
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
_ = v.delete()
return errors.Err(err)
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
@@ -355,7 +364,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
if err != nil {
return nil, err
}
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer, v.walletLock)
}
func (v *YoutubeVideo) Size() *int64 {
@@ -372,13 +381,14 @@ type SyncParams struct {
Fee *sdk.Fee
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
if reprocess && existingVideoData != nil && existingVideoData.Published {
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, err
return summary, errors.Prefix("upgrade failed", err)
}
return v.downloadAndPublish(daemon, params)
}
@@ -499,6 +509,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
Fee: fee,
}
v.walletLock.RLock()
defer v.walletLock.RUnlock()
if v.mocked {
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: streamCreateOptions,
-152
View File
@@ -1,152 +0,0 @@
import os
import sys
from decimal import Decimal
from bitcoinrpc.authproxy import AuthServiceProxy
from lbryum.wallet import Wallet, WalletStorage
from lbryum.commands import known_commands, Commands
from lbryum.simple_config import SimpleConfig
from lbryum.blockchain import get_blockchain
from lbryum.network import Network
def get_lbrycrdd_connection_string(wallet_conf):
settings = {"username": "rpcuser",
"password": "rpcpassword",
"rpc_port": 9245}
if wallet_conf and os.path.exists(wallet_conf):
with open(wallet_conf, "r") as conf:
conf_lines = conf.readlines()
for l in conf_lines:
if l.startswith("rpcuser="):
settings["username"] = l[8:].rstrip('\n')
if l.startswith("rpcpassword="):
settings["password"] = l[12:].rstrip('\n')
if l.startswith("rpcport="):
settings["rpc_port"] = int(l[8:].rstrip('\n'))
rpc_user = settings["username"]
rpc_pass = settings["password"]
rpc_port = settings["rpc_port"]
rpc_url = "127.0.0.1"
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
class LBRYumWallet(object):
def __init__(self, lbryum_path):
self.config = SimpleConfig()
self.config.set_key('chain', 'lbrycrd_main')
self.storage = WalletStorage(lbryum_path)
self.wallet = Wallet(self.storage)
self.cmd_runner = Commands(self.config, self.wallet, None)
if not self.wallet.has_seed():
seed = self.wallet.make_seed()
self.wallet.add_seed(seed, "derp")
self.wallet.create_master_keys("derp")
self.wallet.create_main_account()
self.wallet.update_password("derp", "")
self.network = Network(self.config)
self.blockchain = get_blockchain(self.config, self.network)
print self.config.get('chain'), self.blockchain
self.wallet.storage.write()
def command(self, command_name, *args, **kwargs):
cmd_runner = Commands(self.config, self.wallet, None)
cmd = known_commands[command_name]
func = getattr(cmd_runner, cmd.name)
return func(*args, **kwargs)
def generate_address(self):
address = self.wallet.create_new_address()
self.wallet.storage.write()
return address
class LBRYcrd(object):
def __init__(self, lbrycrdd_path):
self.lbrycrdd_conn_str = get_lbrycrdd_connection_string(lbrycrdd_path)
def __call__(self, method, *args, **kwargs):
return self.rpc(method)(*args, **kwargs)
def rpc(self, method):
return AuthServiceProxy(self.lbrycrdd_conn_str, service_name=method)
def get_wallet_path():
cwd = os.getcwd()
wallet_path = os.path.join(cwd, "wallet.json")
if not os.path.exists(wallet_path):
return wallet_path
i = 1
while True:
wallet_path = os.path.join(cwd, "wallet_%i.json" % i)
if not os.path.exists(wallet_path):
return wallet_path
i += 1
def coin_chooser(lbrycrdd, amount, fee=0.001):
def iter_txis():
unspent = lbrycrdd("listunspent")
unspent = sorted(unspent, key=lambda x: x['amount'], reverse=True)
spendable = Decimal(0.0)
for txi in unspent:
if spendable >= amount:
break
else:
spendable += txi['amount']
yield txi
if spendable < amount:
print spendable, amount
raise Exception("Not enough funds")
coins = list(iter(iter_txis()))
total = sum(c['amount'] for c in coins)
change = Decimal(total) - Decimal(amount) - Decimal(fee)
if change < 0:
raise Exception("Not enough funds")
if change:
change_address = lbrycrdd("getnewaddress")
else:
change_address = None
print "Total: %f, amount: %f, change: %f" % (total, amount, change)
return coins, change, change_address
def get_raw_tx(lbrycrdd, addresses, coins, amount, change, change_address):
txi = [{'txid': c['txid'], 'vout': c['vout']} for c in coins]
txo = {address: float(amount) for address in addresses}
if change_address:
txo[change_address] = float(change)
return lbrycrdd("createrawtransaction", txi, txo)
def main(count, value=None, lbryum_path=None, lbrycrdd_path=None):
count = int(count)
lbryum_path = lbryum_path or get_wallet_path()
if sys.platform == "darwin":
default_lbrycrdd = os.path.join(os.path.expanduser("~"),
"Library/Application Support/lbrycrd/lbrycrd.conf")
else:
default_lbrycrdd = os.path.join(os.path.expanduser("~"), ".lbrycrd/lbrycrd.conf")
lbrycrdd_path = lbrycrdd_path or default_lbrycrdd
l = LBRYcrd(lbrycrdd_path=lbrycrdd_path)
s = LBRYumWallet(lbryum_path)
value = value or 1.0
value = Decimal(value)
coins, change, change_address = coin_chooser(l, count * value)
addresses = [s.generate_address() for i in range(count)]
raw_tx = get_raw_tx(l, addresses, coins, value, change, change_address)
signed = l("signrawtransaction", raw_tx)['hex']
txid = l("sendrawtransaction", signed)
print txid
if __name__ == "__main__":
args = sys.argv[1:]
main(*args)
+414 -15
View File
@@ -71,6 +71,8 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
return sanitizedTags, nil
}
const TagMaxLength = 255
func normalizeTag(t string) (string, error) {
t = strings.ToLower(t)
multipleSpaces := regexp.MustCompile(`\s{2,}`)
@@ -87,7 +89,10 @@ func normalizeTag(t string) (string, error) {
if weirdChars.MatchString(t) {
log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
return "", nil
}
if len(t) > TagMaxLength {
log.Debugf("tag '%s' is too long, skipping\n", t)
return "", nil
}
return t, nil
}
@@ -140,23 +145,417 @@ func (ts *tagsSanitizer) add() {
}
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
DavidPakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
anupjanakpur = "UC_5tRfC4L2AbTz6mj6vZrKw"
PraveenMedia = "UC_fjE70lKNwM9AKIofQv-bA"
MisteriosDesvelados = "UC-FzxivscjYzonBDXgX2GLg"
kaipenchery = "UC-MU4K3Ghl-IdEX4J68tnTA"
Dhinchakpooja = "UC-stzLwoQF_R8Vnfb28d7Lg"
Karolajn = "UC-vYDJQiabg9BK8XTDPFspg"
thefantasio974 = "UC0GHeUcPxfNEZqbFyxy2frQ"
EduPrimitivo = "UC0odmP6ffEw3iVwTEyt-UKA"
isupportanupriyapatel = "UC0sxOzdHnmauMdOra0cxXVg"
theconfusedindian = "UC12ZPYxQbMA1loZE1OA609w"
guriaarts = "UC1fHp166o1Hd024fg3-d-jg"
minutodafisica = "UC1lUEcHrQwbusC5ext1himg"
khabir = "UC1yzNCXXk1h_tHgsqChE8mw"
OMapadaMinaMara = "UC2WMUPTbxQQ9hXq2_uutspg"
oliverjanich = "UC3cmEfpy4XED7YYEe69nIMA"
shyamsadhu = "UC3XAT8oBjL2RaqfblESHW_A"
EbaduRahmantech = "UC4950gpY6Qw1lCAcN6gNWwQ"
_1975oles = "UC4bGQWN4C8idymxlZxw77oA"
_8dsongsbollywood = "UC4nH7zmw41lRDbaiIddumjw"
EYESTV = "UC50CIbyHMydEEzEhV1ZNEBw"
Nono = "UC5yufoRPJy-1e1pR73UTxFQ"
lichtle = "UC6n_2v7YjwZ65F9h77nZPOA"
elabuelokraken = "UC7iWk2xziMR4hs_NzfsYytw"
cidvela = "UCaVOx5GCcSi2ELjWUdf05PA"
jayaskumar = "UCax9CJCQ6aY0bntP3mcIWKQ"
canaldarippi = "UCB_fba7yYMwa91F7rijBsVA"
FunMusicClassical = "UCB_X256IN7QiBtDkueo_6QQ"
minutodaterra = "UCB0zinWfy-dS_NqcOINYo3A"
franciscoalves = "UCbAm6YcZGk04obnFb-LuOSA"
criptomonedastv = "UCbK6Awel1-o-9JDVMFq808Q"
lux77 = "UCBYm4l3NX352goFtjSgZ7jA"
MrLuckyOficial = "UCEbMhGhZ1JoOYKgbgVyeQaA"
KotneKit = "UCf_1BQz3T151Eph0Sarj3Yw"
jaysgaming = "UCFAkedtc3jjDZQqqIULxqWg"
iamdineshthakur = "UCg0VInmUZoSdHD-DNmjifBA"
Musicreationz = "UCGJEtZB0Nj3rT3zZfyfjIZQ"
lafenixtv = "UCHAx8o0jj-8L_sqR5QM1Cwg"
guardeiafe = "UCHWjmG-Ce93VNcNZMMUyGQQ"
KHANEKIKHUSHBOO = "UCI8DRKcrfrHklRqSJHPhOwQ"
AjayGamingYT = "UCi9r5igvQblbI0PhJFj-IDg"
SzekelyVegan = "UCiqvKH1Ib_VPwh9bQf_hlXQ"
barzoy888 = "UCIYF1orTg4nDvv29ORoKXyA"
_7playstudiopersian = "UCJ3QUKcd9yOhiJzZBR2zT3Q"
eazypurple = "UCJ6Hd8S39g7pe65A6k42NZw"
GennadyM = "UCjZlOrmC7hi_1kMA6BKZXeQ"
GTGujarati = "UCKnYZmYFdFX4hP09Hh8k8jw"
annapavlidou = "UCKPSHyCFIDtMqzbJ_Qx-7aw"
Tamilmemesclub = "UCL_DPW38jcSGWy0Fu0A_hCg"
QuickReactionTeam = "UCl--LVcANGKHJYN7O8uY6OA"
rodrigojoseoficial = "UClMsSb57Jvexl1a0a2c2_CQ"
AzadChaiwala = "UCM5QNdoIefx6eumjPk8ZTMw"
lafenixextras = "UCM7zougvyPtfd9zw5HmAGlg"
JustSawAGhost = "UCmQYXE03n6qlF9FhtAPztcw"
mirchibangla = "UCmzj6hXrPZ_AwIZ8lgo-HuQ"
marsaguirre = "UCN3OMvdU7ySvE6meGjo9omg"
JustGoool = "UCn7kTV_syU_rI0jANxrpaFg"
nossocanal = "UCNDb9jdx5Jz-C7hWJv3y6Fw"
CSNN = "UCnfn-8PJbjYFAJ1fqzMdcew"
pedronedved = "UCNhb2Fz3sVLZcMb_arm1cMQ"
famoosh = "UCnQU2QZLVG1wxHoy_9LZucA"
Cenoritas = "UCo-VSGoYy_4IjXF9u4QqTiw"
fantabobgames = "UCo0U1tbk3YbqiLDhkeWOviQ"
thesoundofsoul = "UCOey6R7Ktnil8RZrY_xJYiA"
SaurabhMishraJournalist = "UCoHVCBoSfUOlQVa80dlN60g"
YoYoHoneySinghlive = "UCOZTp3nGj39-snzo4QSdduA"
CINEGLITZ = "UCP_p8JkLOwPcGAnxN1iVhZA"
lovetreta = "UCPi3uuGrh2mxnZJMfRhcjiw"
canalvendimeusofa = "UCpk58NDdaKdX0QiiA2e79tg"
CARIBEANVIDEOS = "UCPynGkfzH35RKyXRccCgzNg"
Dimon = "UCpZsZZ6KEqCHeAJ_Y0gEbyQ"
Anantvijaysoni = "UCQ3NeFKF2yo70xxBBUDeXXQ"
canalilhadebarbados = "UCR5L_Q8Tiljy7WKEQOrGKbg"
SotomayorTV2 = "UCR70CjRHxQilfEUgBBYpDhg"
Recipesarab = "UCRbMDfkH_bPUjQsv5dHcFQA"
_4ak10 = "UCsbrIjDPPXuVlWApYTnQECA"
Top50 = "UCSjXR8uvU4PijMM_kgyVQfw"
SoumyajitPyne = "UCSJXwgF3SePNfyojYG3yt6Q"
SneksShow = "UCsqZSkVccnyIxSE6pZuj13g"
casadosaber = "UCtvvTFp0XANyllOdmzZr9VQ"
Entarexyt = "UCuDkq6yAb5zdM2zIRG6VRDw"
UCurU6GLM4ggcLtWWAOTzlYw = "UCurU6GLM4ggcLtWWAOTzlYw"
promining = "UCviqKxlMnZqBipTzwixkGyg"
zapkids = "UCx2uGhYa9EuCNmbL72hVKGg"
Canalokok = "UCXyCAcjoWz9SMLGJT8dR0pA"
SmoothGames = "UCY8gvG25rOZ4hWWvcQH88PQ"
LIGATFA = "UCYhiIMOlDn_HEn3zvqZCDiw"
DeHamad = "UCZ5G07Vw7IaV81InBgYLCcg"
misszizi = "UCE55WTFs4ekJ_aWCoNEapbQ"
creationshub = "UCNfELkowZPIQ-Vegmb3pIBQ"
AlicePandora = "UCScRxEtwlt082_6ThI0YJbg"
Akito0405 = "UCXHWx1teSYIKwTGiscYaC-Q"
dashcambristol = "UCD0IC8bZI-MfIutkgPHlyWA"
adrianbonillap = "UCdmY7p_kC-QN8jS7rocmCSA"
SubhamVlogs = "UCXtdoCLRlnLIW3Skix0rHaQ"
DisciplesofJesusChristJeremiahPayne = "UC_y2rVsotcQcVF7LImVGs5Q"
ModernGalaxy = "UCGISiGs_RL7Z1qfs1-GD6PA"
socofilms = "UCyDS9p6NWHpU9XbbbYLFLBw"
buriedone = "UC_lm7xXB3adOTc0T1FwyGRQ"
bitcoin = "UC-f5nPBEDyUBZz_jOPBwAfQ"
globalrashid = "UC2ldcEtbR7cFYadgrnW3B6Q"
gameofbitcoins = "UC2WKsYBxMwx7E7ENkND-fkg"
CryptoInsightsBrasil = "UC4BrnREinCBUenQZi4ZU95w"
Crypt0 = "UCdUSSt-IEUg2eq46rD7lu_g"
cryptomined = "UCGQ3XHtsH8Q9iQr9bFbgfDA"
altcoinbuzz = "UCGyqEtcGQQtXyUwvcy7Gmyg"
crypto = "UCiMgF08KQ4z-Gnu8o2BLOxA"
Crypto99 = "UCjsrdOJCAKcuBqyyqjg1cCQ"
TheCryptoLark = "UCl2oCaw8hdR_kbqyqd2klIA"
NuggetsNews = "UCLo66QVfEod0nNM_GzKNxmQ"
btckyle = "UCNCGCxxTT10aeTgUMHW5FfQ"
LouisThomas = "UCpceefaJ9vs4RYUTsO9Y3FA"
BitsBeTrippin = "UCVVWXoQfMfQVuzxcLylq9aA"
cryptocrow = "UCwsRWmIL5XKqFtdytBfeX0g"
KouSuccessLeeFX = "UC0YkP4Fg_d8y6JFwLj3MLdg"
TheSchiffReport = "UCIjuLiLHdFxYtFmWlbTGQRQ"
MRU = "UCnkEhPBMZcEO0QGu51fDFDg"
Vidello = "UCwMaWqZ6SdDpTaYOW0huELw"
kaccreative = "UC02O9ICMuwrfULSa0N6SiSw"
shecooks = "UChZYqTJkeYV2r7WETcytdPQ"
vegsource = "UClEsPxvotpTJ1Z8eu2Y97rg"
VeganGains = "UCr2eKhGzPhN5RPVk5dd5o3g"
AwesomeKnowledge = "UC_pC4T8vr-caiNDYU6PYTTA"
luckyloush = "UCbO0Oomf_jr20Wb4V2ap5Uw"
MichaelLuzzi = "UCDwLrj4DSGrw3gFvANbl_Cw"
NakedApe = "UCMOaRU-YsXVgU-WahBkZqWQ"
ParaReact = "UCuJKELjsmWTlJG0X7T4ZD_A"
dullytv = "UCv1J91Nhn7KsxMFaxKChT3w"
REDONKULAS = "UCwd_sSDZ8EQt6SEeOO2tBRA"
RedactedTonight = "UCyvaZ2RHEDrgKXz43gz7CbQ"
TonyTornado = "UCZu9AV3mrCCDpK_dy1qJRAQ"
hoodlumscrafty = "UC0cTVjYKgAnBrXQKcICyNmA"
Karmakut = "UC2B8TOklu2rSDULqAzwn5GQ"
Draxr = "UC72o3j23E1wKskBDEKlmTOw"
BigfryTV = "UC7FVdUA3SxDMfl4fLBGlADg"
CSGONews = "UC7L6NRLyldvxWukhOHABazQ"
ZedGaming = "UC7Q15H71DmB7F1iSFE1Z8LQ"
KamiVS = "UC9fh15yUcGAr8iUfQaoRRpQ"
grabthegames = "UCaJFEgY6ij05Fxgn6qtcX_g"
blueplays = "UCbjMsFlYb2NLpjS3uDzm9ow"
ImNotBonkers = "UCCyN0G77B7wnf-1AOTI3gWQ"
GunslingerMedia = "UCDjs4JoXmMzvaZyPVsvjKXw"
LeagueVoices = "UCdkonRjBLzr1Adf7Jhu1bWQ"
SirPugger = "UCelqWKTcCvBZP_1_1iYZAWA"
gamesoup = "UCGPMrF9AN_D9BrmSmMeV3hA"
ProHenis = "UCIjFoXSQ9HYbcWmRVIsH-Ow"
Dota2Divine = "UCiR9IHCurqVHpC821NVcW6g"
CrazyFoxMovies = "UCjewtQLpJEENPLbrCtb6YpA"
Larry = "UCJVdNvvuvOnthuWVQjYff2w"
hottake = "UCK24784Oqb4oYQfHL9XePNg"
Breezy = "UCKWRpZpcLKriWd1am9SHf8A"
retrorgb = "UCLPIbBCKVH2uKGm5C4sOkew"
Zer0Gamer = "UCmII34jN4rqCIsGqWkK5JEg"
nickatnyte = "UCMxYQX1zaepCgmiSmwbT39w"
DyllonStejGaming = "UCngaLL0QDbsAGYj7zsB8o_Q"
bluedrake42 = "UCNSwcDEUfIEzYdAPscXo6ZA"
Rerez = "UCoFpRCAsKfWAshvLE1bYzdw"
Op = "UCowi5kFfvGXR8NqhyE6jneQ"
kidsgamesfun = "UCqjGzmb2pMWSFYm0GzKeTEA"
GamesGlitches = "UCRj3Q06KOxAZWHaHScrkAOw"
Musikage = "UCsej4tgCoXDgVH3J7M3NMgw"
JeffyGaming = "UCTZzSNnZ43XQslejT5wFdRw"
oniblackmage = "UCUEF9XL3o8dZ6hvVf8jAi8Q"
nickatnyte2 = "UCuoTqrobMyZj0ge8LOCkiSw"
KazeEmanuar = "UCuvSqzfO_LV_QzHdmEj84SQ"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
GaminGHD = "UCW-thz5HxE-goYq8yPds1Gw"
BHGaming = "UCX4N3DioqqrugFeilxTkSIw"
Potato = "UCxPPTDNH85HZWxrgZ3FQBYA"
MikeyTaylorGaming = "UCycXj6lRWtsSqo-bZOIZePw"
Acituanbus = "UCzfx1QvKjn-BxLMLBdBGgMw"
juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
KhanAcademy = "UC4a-Gbdw7vOaccHmFo40b9g"
DON = "UCAYrPk70AePJZSaVLKrWdfQ"
shogogarcia = "UCE3yZjxDg3iI91bcNJDFnsg"
alphalifestyleacademy = "UCeggEaXtJu2domMahYMD_ig"
NileRed = "UCFhXFikryT4aFcLkLw2LBLA"
veritasium = "UCHnyfMqiRRG1u-2MsSQLbXA"
stevecronin = "UCJYawZQYwjZ76mrF_US9eNg"
jeranism = "UCS_FY5mR4g22L_E9t1D_ExQ"
MinutePhysics = "UCUHW94eEFW7hkUMVaZz4eDg"
_3Blue1Brown = "UCYO_jab_esuFRV4b17AJtAw"
Itsrucka = "UC-B2LyEZcl3avG0coKeohGQ"
unitedtaps = "UC5Q8e9-uutVZmiwAcEHKuzA"
srodalmenara = "UC9kc1DaOy2kSzXHZgpn9kfQ"
ShutupAndPlay = "UCAwuvzhah0KUw5QNihSkEwQ"
sanx2 = "UCJ_waKl9kjbhfXI3LJOfHvA"
DerickWattsAndTheSundayBlues = "UCmZhiZq7M7d73Kbey4yna_Q"
akirathedon = "UCsoiSpBvkr4Y-78Pj3recUw"
Musicoterapia = "UCsoSK8K4OpdMV1tqJFwO5QA"
daydreaming = "UCtbuGylbRXc42pIxWey19Dg"
RemixHolicRecords = "UCUW5GjwcXgbPcfk00t-GQZA"
EDMBot = "UCvmUdL2NHWlj1NRiNJPI-TQ"
TrioTravels = "UCdAPAmdnkdFsH5R2Hxevucg"
caosonnguyen = "UCwPeW9kFId5-VbQ2LQEjVhg"
TheAlmonteFilms = "UC4C_SF5koS4Q5om50b9NMTw"
timcast = "UCG749Dj4V2fKa143f8sE60Q"
SeekingTheTruth = "UCHrDpTVL9S0h91u9UCPgVbA"
davidpakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
mikenayna = "UCzk08fzh5c_BhjQa1w35wtA"
DoorMonster = "UC-to_wlckb-bFDtQfUZL3Kw"
barnacules = "UC1MwJy1R0nGQkXxRD9p-zTQ"
brightsunfilms = "UC5k3Kc0avyDJ2nG9Kxm9JmQ"
Onision = "UC5OxQNCgW88FDBxeZCnrBbg"
top10archive = "UCa03bf8gAS2EtffptV-_jfA"
thought = "UCb0yiUQhhLV_jpY3BayJaLA"
MothersBasement = "UCBs2Y3i14e1NWQxOGliatmg"
VlogsOfKnowledge = "UCDUPGR6uL5uz0_hiRAQjZMQ"
GorTheMovieGod = "UCHdTVw89QU6coU1MgN-9RHA"
iamalexoconnor = "UCKAQLEk1GGqnPtov9EW0huQ"
ADedits = "UCsX-zRuq3ovMsgFqEQLw2Bw"
SynthCool = "UCxGTHsD0pLSFlFI7M7jYmBQ"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
Sciencedocumentinhindi = "UC9SpfUF3rm-MGep5WE6FSCA"
NurdRage = "UCIgKGGJkt1MrNmhq3vRibYA"
MINDBLASTER = "UC_ZMqbRu44jK-EogjYyHz8g"
DaminousPurity = "UCdKRWvz50QoioZFgu6Nf9og"
ScammerRevolts = "UC0uJKUXiU5T41Fzawy5H6mw"
eevblog = "UC2DjFE7Xf11URZqWBigcVOQ"
Luke = "UC2eYFnH61tmytImy1mTYvhA"
AppGirl = "UC389S4_2Yt9cei1qNDwmBrA"
thecryptodad = "UC68x_TIzqCtF69fYl2_kl3w"
ChrisWereDigital = "UCAPR27YUyxmgwm3Wc2WSHLw"
alecaddd = "UCbmBY_XYZqCa2G0XmFA7ZWg"
EliTheComputerGuy = "UCD4EOyXKjfDUhCI6jlOZZYQ"
archetapp = "UCDIBBmkZIB2hjBsk1hUImdA"
PCPlaceNZ = "UCf5ZTSZAKbinY03jOwylfOg"
NaomiSexyCyborgWu = "UCh_ugKacslKhsGGdXP0cRRA"
NibiruWatcher = "UCi62JvN-lUn7hVL3ffofADA"
imineblocks = "UCjYHcWGAjUVqU49D2JOKD3w"
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
CooLoserTech = "UCl97rZ2Tc7KV9lktmmHNFDQ"
TechHD = "UCN3bPy04Jkp3ADRtyYvXomQ"
MiketheScrapper = "UCqtlJpkH_llXS_vuDExGVvw"
eevblog2 = "UCr-cm90DwFJC0W3f9jBs5jA"
thecreativeone = "UCTikFhzCiIXfOMS7D29dvYg"
weekendtricks = "UCYtAJXx0ymGPpCndn2Gt6-w"
GBGuns = "UC2VOURrALs1CwVmbGlXJOPQ"
Matsimus = "UCFWjEwhX6cSAKBQ28pufG3w"
TheLateBoyScout = "UCZjvj5MN3BMxPFfdEKIrvxQ"
BravoCinematografica = "UC2ruSXQoKMgr7JXzwO0H0KA"
PyroNation = "UC4ffy3n1hE7Z8q-2KVq_91Q"
dramatuber = "UC4Y8mImty3gFEG9grsR6T-w"
BarnabasNagy = "UC8TRZRK1sJfxKJ0tXMmGTow"
avery = "UCcfjIZLDCuSqkIlkH8i4DDg"
Tingledove = "UCfI7wtV6K64gVbzjH7DOA_Q"
crmjewelers = "UChpFWeF84jA5JeV3YyIo3pQ"
NerfNerd18 = "UCIgmaEJNqvH9bU9zGMapZGg"
SEIJIHITO = "UCNqUrLE6dI8fWw_u3HQkpXA"
anvithavlogs = "UCsP9pYat2DEBvnvF_iFGG_w"
YoelRekts = "UCZ_BcFyhIo6GdtTSrqXXepg"
TechFox = "UCIp-oTSdFO7BhAJpW2d5HMQ"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
Juggling: {"juggling", "circus arts", "malabares"},
JustJuggling: {"juggling", "circus arts", "malabares"},
JordanBPeterson: {"postmodernism", "psychology", "news"},
DavidPakman: {"news"},
TheLinuxGamer: {"gaming", "technology", "linux"},
JustJuggling: {"juggling", "circus arts", "malabares"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
TechFox: {"technology", "reviews"},
misszizi: {"art", "pop culture"},
creationshub: {"art"},
AlicePandora: {"art"},
Akito0405: {"art"},
dashcambristol: {"automotive"},
adrianbonillap: {"automotive"},
SubhamVlogs: {"automotive"},
DisciplesofJesusChristJeremiahPayne: {"beliefs"},
ModernGalaxy: {"beliefs"},
socofilms: {"beliefs"},
buriedone: {"blockchain"},
bitcoin: {"blockchain"},
globalrashid: {"blockchain"},
gameofbitcoins: {"blockchain"},
CryptoInsightsBrasil: {"blockchain"},
Crypt0: {"blockchain"},
cryptomined: {"blockchain"},
altcoinbuzz: {"blockchain", "technology"},
crypto: {"blockchain"},
Crypto99: {"blockchain"},
TheCryptoLark: {"blockchain", "technology"},
NuggetsNews: {"blockchain", "learning"},
btckyle: {"blockchain"},
LouisThomas: {"blockchain"},
BitsBeTrippin: {"blockchain"},
cryptocrow: {"blockchain"},
KouSuccessLeeFX: {"economics", "learning"},
TheSchiffReport: {"economics"},
MRU: {"economics"},
Vidello: {"economics", "pop culture"},
kaccreative: {"food", "art"},
shecooks: {"food"},
vegsource: {"food"},
VeganGains: {"food"},
AwesomeKnowledge: {"funny"},
luckyloush: {"funny"},
MichaelLuzzi: {"funny", "pop culture"},
NakedApe: {"funny"},
ParaReact: {"funny"},
dullytv: {"funny"},
REDONKULAS: {"funny"},
RedactedTonight: {"funny", "news"},
TonyTornado: {"funny"},
hoodlumscrafty: {"gaming", "pop culture"},
Karmakut: {"gaming"},
Draxr: {"gaming", "pop culture"},
BigfryTV: {"gaming"},
CSGONews: {"gaming", "funny"},
ZedGaming: {"gaming"},
KamiVS: {"gaming"},
grabthegames: {"gaming"},
blueplays: {"gaming"},
ImNotBonkers: {"gaming", "funny"},
GunslingerMedia: {"gaming"},
LeagueVoices: {"gaming"},
SirPugger: {"gaming"},
gamesoup: {"gaming"},
ProHenis: {"gaming"},
Dota2Divine: {"gaming"},
CrazyFoxMovies: {"gaming", "pop culture", "technology"},
Larry: {"gaming"},
hottake: {"gaming", "funny"},
Breezy: {"gaming", "funny"},
retrorgb: {"gaming"},
Zer0Gamer: {"gaming"},
nickatnyte: {"gaming"},
DyllonStejGaming: {"gaming"},
bluedrake42: {"gaming"},
Rerez: {"gaming"},
Op: {"gaming"},
kidsgamesfun: {"gaming", "pop culture"},
GamesGlitches: {"gaming"},
Musikage: {"gaming", "pop culture"},
JeffyGaming: {"gaming", "funny"},
oniblackmage: {"gaming"},
nickatnyte2: {"gaming"},
KazeEmanuar: {"gaming"},
TheLinuxGamer: {"gaming", "technology", "linux"},
GaminGHD: {"gaming", "pop culture"},
BHGaming: {"gaming"},
Potato: {"gaming"},
MikeyTaylorGaming: {"gaming"},
Acituanbus: {"gaming"},
juggling: {"juggling", "circus art", "malabares"},
KhanAcademy: {"learning", "science"},
DON: {"learning", "pop culture"},
shogogarcia: {"learning"},
alphalifestyleacademy: {"learning"},
NileRed: {"learning", "science"},
veritasium: {"learning", "science"},
stevecronin: {"learning"},
jeranism: {"learning"},
MinutePhysics: {"learning", "science"},
_3Blue1Brown: {"learning"},
Itsrucka: {"music"},
unitedtaps: {"music"},
srodalmenara: {"music"},
ShutupAndPlay: {"music", "learning"},
sanx2: {"music"},
DerickWattsAndTheSundayBlues: {"music", "funny"},
akirathedon: {"music"},
Musicoterapia: {"music"},
daydreaming: {"music"},
RemixHolicRecords: {"music"},
EDMBot: {"music"},
TrioTravels: {"nature"},
caosonnguyen: {"nature"},
TheAlmonteFilms: {"news"},
timcast: {"news", "technology"},
SeekingTheTruth: {"news"},
davidpakman: {"news"},
mikenayna: {"news"},
DoorMonster: {"pop culture", "funny"},
barnacules: {"pop culture", "gaming"},
brightsunfilms: {"pop culture"},
Onision: {"pop culture", "funny"},
top10archive: {"pop culture"},
thought: {"pop culture", "learning"},
MothersBasement: {"pop culture", "gaming"},
VlogsOfKnowledge: {"pop culture", "gaming"},
GorTheMovieGod: {"pop culture"},
iamalexoconnor: {"pop culture"},
ADedits: {"pop culture"},
SynthCool: {"pop culture", "funny"},
JordanBPeterson: {"psychology", "postmodernism", "news"},
Sciencedocumentinhindi: {"science"},
NurdRage: {"science", "learning"},
MINDBLASTER: {"sports", "funny"},
DaminousPurity: {"sports", "gaming"},
ScammerRevolts: {"technology"},
eevblog: {"technology", "learning"},
Luke: {"technology", "funny"},
AppGirl: {"technology"},
thecryptodad: {"technology", "blockchain"},
ChrisWereDigital: {"technology"},
alecaddd: {"technology", "learning"},
EliTheComputerGuy: {"technology"},
archetapp: {"technology", "learning"},
PCPlaceNZ: {"technology"},
NaomiSexyCyborgWu: {"technology"},
NibiruWatcher: {"technology", "learning"},
imineblocks: {"technology", "blockchain"},
Lunduke: {"technology"},
CooLoserTech: {"technology"},
TechHD: {"technology", "learning"},
MiketheScrapper: {"technology"},
eevblog2: {"technology"},
thecreativeone: {"technology", "gaming"},
weekendtricks: {"technology"},
GBGuns: {"weapons"},
Matsimus: {"weapons", "gaming"},
TheLateBoyScout: {"weapons"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,
+5 -4
View File
@@ -61,10 +61,11 @@ func (u *thumbnailUploader) uploadThumbnail() error {
uploader := s3manager.NewUploader(s3Session)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
})
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)