Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
74d5e2606d | ||
|
|
b8d2b9335b | ||
|
|
637785f4ac | ||
|
|
c9dffb0a82 | ||
|
|
6ea0bd9834 | ||
|
|
93986c53e0 | ||
|
|
843575c4ba | ||
|
|
aecfc67f84 | ||
|
|
1f7017f3d0 | ||
|
|
75364805a5 | ||
|
|
07f66a089f | ||
|
|
36abd1e471 | ||
|
|
502bfa924b | ||
|
|
23b8fb4e54 | ||
|
|
a648287e1c | ||
|
|
af13ec5684 |
No files matched your search
+1
-1
@@ -9,7 +9,7 @@ env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "+G0TxyGyRYJb0L0qCCn6NPregigwhalkS9+/3mkV41xoAwUlEU9aclSJQVKKqeMrV0PkmJ6KrzYD78tnRj8Q5ZDe93LBDbc/WhXZ91JbRmIXIsVktz/78R2a2Q0DBrXMlyWXS84QouRfI3eyxf2BusERSYdr3H6BL4r15PIHJEmLGOchHsHwSI46PRGag+IFnbUb+fyUSs7er6gKeSp4+ZK1bSUgxiMUQrZfImYI9LS+cgqpDh67pX5jRbDNBihZZUwfreToYw3RbIAom4i2oix1rBBD/jpcTwO+WdyXFtur8RYGnsknePU88KiAB9bcXU4T2xVxR9wusK6jX5G2BChEOG9PjuVfeDqXBnskOEtAevwAKJRYW6ATmXqQZ+TAy1DFxsUWPVS7ykIA+69n7HzE7xu50AB1U2n6myT977i4jhBzw3VrB9NQz//4b2j6MCG1PhIgB38kffQWVsa/CF4RiiWi6a7ePRsPP0HdRUbXFQJWPevKgVJ9ve/ArQKiea1rP9zy9KdFAP30vESBkn4WMOCjfg7F6JbfV3e1ywBq9XMznj14iAZHwXR1tpPp0GekuMlVJHJWa0Ea/dfDGWshnr1Be71vc9T118w7sd0v1Dp4EQCFa4B0Hx7Ual7uWgeuweL/JLr1Am3BgWVcXOWrhzMT0b1tGFa9ZNz2Wwk="
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
deploy:
|
||||
provider: script
|
||||
skip_cleanup: true
|
||||
|
||||
@@ -5,8 +5,6 @@ require (
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/PuerkitoBio/goquery v1.5.0 // indirect
|
||||
github.com/aws/aws-sdk-go v1.17.3
|
||||
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b // indirect
|
||||
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 // indirect
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/go-ini/ini v1.42.0 // indirect
|
||||
github.com/golang/protobuf v1.3.1 // indirect
|
||||
@@ -17,8 +15,7 @@ require (
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
|
||||
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18
|
||||
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 // indirect
|
||||
github.com/lbryio/lbry.go v1.0.6
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
@@ -37,7 +34,6 @@ require (
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
|
||||
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be // indirect
|
||||
google.golang.org/api v0.3.2
|
||||
google.golang.org/appengine v1.5.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
|
||||
|
||||
@@ -9,6 +9,7 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
|
||||
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
@@ -21,19 +22,17 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
|
||||
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
|
||||
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
|
||||
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b h1:7J7sEce3LgtbMgs7PKcN61gF3b4txM6SjaRoJTSk640=
|
||||
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 h1:GbQHMJ2u/geMPV1tbN7i7zARSoPAPuXWa44V0KYvJXU=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
@@ -101,9 +100,11 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
|
||||
@@ -112,6 +113,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
@@ -125,13 +127,13 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18 h1:ZhWjtvaq5r7julhcF9OSgx4bLv9UsdIx27zH1/fbrEc=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18/go.mod h1:kd08aOMCuBVYJ3EafY4Kx3dRAWWQYhobJ9beREgsaRI=
|
||||
github.com/lbryio/lbry.go v1.0.6 h1:dGZHwYdD5bb7PCOhG2ksJuvhOjqIARAcXJGggaOhC6o=
|
||||
github.com/lbryio/lbry.go v1.0.6/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a h1:twWvrsBDvSb+qnmpSq3nvFrodgC5PpXUipyo4T/W790=
|
||||
github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 h1:iERWR8Dkng30eRInI7gzolTEJBW9nBSK/sT+Z9aSipI=
|
||||
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
|
||||
@@ -178,11 +180,11 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
@@ -211,7 +213,6 @@ go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
|
||||
@@ -249,8 +250,8 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be h1:mI+jhqkn68ybP0ORJqunXn+fq+Eeb4hHKqLQcFICjAc=
|
||||
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -279,7 +280,6 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
|
||||
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -287,7 +287,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
|
||||
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
@@ -298,6 +297,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
+21
-12
@@ -2,6 +2,8 @@ package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -70,21 +72,23 @@ func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
@@ -206,7 +210,12 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SyncManager) GetS3AWSConfig() aws.Config {
|
||||
return aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
|
||||
Region: &s.awsS3Region,
|
||||
}
|
||||
}
|
||||
func (s *SyncManager) checkUsedSpace() error {
|
||||
usedPctile, err := GetUsedSpace(s.blobsDir)
|
||||
if err != nil {
|
||||
|
||||
+90
-43
@@ -7,8 +7,6 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
@@ -21,6 +19,27 @@ import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
const minimumRefillAmount = 3
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if os.Getenv("REGTEST") == "true" {
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
}
|
||||
for _, a := range accounts {
|
||||
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
|
||||
ChangeMaxUses: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *Sync) walletSetup() error {
|
||||
//prevent unnecessary concurrent execution
|
||||
s.walletMux.Lock()
|
||||
@@ -42,16 +61,11 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
log.Debugf("Starting balance is %.4f", balance)
|
||||
|
||||
var numOnSource int
|
||||
if s.LbryChannelName == "@UCBerkeley" {
|
||||
numOnSource = 10104
|
||||
} else {
|
||||
n, err := s.CountVideos()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
numOnSource = int(n)
|
||||
n, err := s.CountVideos()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
numOnSource := int(n)
|
||||
|
||||
log.Debugf("Source channel has %d videos", numOnSource)
|
||||
if numOnSource == 0 {
|
||||
@@ -68,9 +82,18 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
|
||||
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
|
||||
if numPublished > numOnSource && balance < 1 {
|
||||
if s.Manager.syncStatus == StatusPendingUpgrade {
|
||||
videosToUpgrade := 0
|
||||
for _, v := range s.syncedVideos {
|
||||
if v.Published && v.MetadataVersion < 2 {
|
||||
videosToUpgrade++
|
||||
}
|
||||
}
|
||||
minBalance += float64(videosToUpgrade) * 0.001
|
||||
}
|
||||
if numPublished > numOnSource && balance < minimumRefillAmount {
|
||||
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
|
||||
minBalance = 1 //since we ended up in this function it means some juice is still needed
|
||||
minBalance = minimumRefillAmount
|
||||
}
|
||||
amountToAdd := minBalance - balance
|
||||
|
||||
@@ -83,8 +106,8 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
|
||||
if amountToAdd > 0 {
|
||||
if amountToAdd < 1 {
|
||||
amountToAdd = 1 // no reason to bother adding less than 1 credit
|
||||
if amountToAdd < minimumRefillAmount {
|
||||
amountToAdd = minimumRefillAmount
|
||||
}
|
||||
err := s.addCredits(amountToAdd)
|
||||
if err != nil {
|
||||
@@ -98,7 +121,7 @@ func (s *Sync) walletSetup() error {
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get unused address")
|
||||
}
|
||||
s.claimAddress = string((*claimAddress)[0])
|
||||
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
|
||||
if s.claimAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
@@ -122,7 +145,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
defaultAccount := ""
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefaultAccount {
|
||||
if account.IsDefault {
|
||||
defaultAccount = account.ID
|
||||
break
|
||||
}
|
||||
@@ -246,9 +269,11 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
channelUsesOldMetadata := false
|
||||
if len((*channels).Items) == 1 {
|
||||
channel := ((*channels).Items)[0]
|
||||
if channel.Name == s.LbryChannelName {
|
||||
channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
|
||||
//TODO: eventually get rid of this when the whole db is filled
|
||||
if s.lbryChannelID == "" {
|
||||
err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
|
||||
@@ -256,7 +281,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
s.lbryChannelID = channel.ClaimID
|
||||
return err
|
||||
if !channelUsesOldMetadata {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
|
||||
}
|
||||
@@ -290,7 +317,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet").Id(s.YoutubeChannelID).Do()
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
@@ -300,24 +327,23 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
channelInfo := response.Items[0].Snippet
|
||||
channelBranding := response.Items[0].BrandingSettings
|
||||
|
||||
thumbnail := channelInfo.Thumbnails.Default
|
||||
if channelInfo.Thumbnails.Maxres != nil {
|
||||
thumbnail = channelInfo.Thumbnails.Maxres
|
||||
} else if channelInfo.Thumbnails.High != nil {
|
||||
thumbnail = channelInfo.Thumbnails.High
|
||||
} else if channelInfo.Thumbnails.Medium != nil {
|
||||
thumbnail = channelInfo.Thumbnails.Medium
|
||||
} else if channelInfo.Thumbnails.Standard != nil {
|
||||
thumbnail = channelInfo.Thumbnails.Standard
|
||||
}
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, ""),
|
||||
Region: &s.AwsS3Region,
|
||||
})
|
||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bannerURL *string
|
||||
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bannerURL = &bURL
|
||||
}
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
@@ -326,16 +352,37 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if channelInfo.Country != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
|
||||
}
|
||||
c, err := s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: channelInfo.Title,
|
||||
Description: channelInfo.Description,
|
||||
Tags: nil,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
})
|
||||
var c *jsonrpc.TransactionSummary
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: channelInfo.Title,
|
||||
Description: channelInfo.Description,
|
||||
Tags: nil,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: channelInfo.Title,
|
||||
Description: channelInfo.Description,
|
||||
Tags: nil,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
CoverURL: bannerURL,
|
||||
})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+51
-67
@@ -1,11 +1,7 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -21,6 +17,7 @@ import (
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
@@ -51,7 +48,7 @@ type video interface {
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, string, float64, string, int, *namer.Namer, float64) (*sources.SyncSummary, error)
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
// sorting videos
|
||||
@@ -319,12 +316,14 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
if *e != nil {
|
||||
//conditions for which a channel shouldn't be marked as failed
|
||||
noFailConditions := []string{
|
||||
"this youtube channel is being managed by another server",
|
||||
"interrupted during daemon startup",
|
||||
"playlist items not found",
|
||||
}
|
||||
if util.SubstringInSlice((*e).Error(), noFailConditions) {
|
||||
return
|
||||
@@ -390,20 +389,24 @@ func logShutdownError(shutdownErr error) {
|
||||
|
||||
var thumbnailHosts = []string{
|
||||
"berk.ninja/thumbnails/",
|
||||
"https://thumbnails.lbry.com/",
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
if c.Value.GetStream().GetThumbnailUrl() == "" {
|
||||
if c.Value.GetThumbnail() == nil || c.Value.GetThumbnail().GetUrl() == "" {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
|
||||
return util.InSlice(c.Value.GetStream().GetThumbnailUrl(), thumbnailHosts)
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fixDupes abandons duplicate claims
|
||||
@@ -414,7 +417,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetStream().GetThumbnailUrl()
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
|
||||
log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID)
|
||||
@@ -450,7 +453,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
|
||||
}
|
||||
count++
|
||||
//check if claimID is in remote db
|
||||
tn := c.Value.GetStream().GetThumbnailUrl()
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
pv, ok := s.syncedVideos[videoID]
|
||||
if !ok || pv.ClaimName != c.Name {
|
||||
@@ -481,7 +484,10 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) doSync() error {
|
||||
var err error
|
||||
err := s.enableAddressReuse()
|
||||
if err != nil {
|
||||
return errors.Prefix("could not set address reuse policy", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
@@ -545,7 +551,7 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
|
||||
if s.LbryChannelName == "@UCBerkeley" {
|
||||
err = s.enqueueUCBVideos()
|
||||
err = errors.Err("UCB is not supported on this version of YTSYNC")
|
||||
} else {
|
||||
err = s.enqueueYoutubeVideos()
|
||||
}
|
||||
@@ -613,6 +619,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -643,8 +650,20 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
}
|
||||
SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
|
||||
}
|
||||
existingClaim, ok := s.syncedVideos[v.ID()]
|
||||
existingClaimID := ""
|
||||
existingClaimName := ""
|
||||
existingClaimSize := v.Size()
|
||||
if ok {
|
||||
existingClaimID = existingClaim.ClaimID
|
||||
existingClaimName = existingClaim.ClaimName
|
||||
if existingClaim.Size > 0 {
|
||||
existingClaimSize = &existingClaim.Size
|
||||
}
|
||||
}
|
||||
|
||||
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, "", "", err.Error(), v.Size())
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
|
||||
if err != nil {
|
||||
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
}
|
||||
@@ -713,14 +732,14 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
||||
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
||||
}
|
||||
req2 := service.Videos.List("snippet,contentDetails").Id(strings.Join(videoIDs[:], ","))
|
||||
req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
|
||||
|
||||
videosListResponse, err := req2.Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position))
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig()))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
@@ -752,55 +771,6 @@ Enqueue:
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) enqueueUCBVideos() error {
|
||||
var videos []video
|
||||
|
||||
csvFile, err := os.Open("ucb.csv")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := csv.NewReader(bufio.NewReader(csvFile))
|
||||
for {
|
||||
line, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
data := struct {
|
||||
PublishedAt string `json:"publishedAt"`
|
||||
}{}
|
||||
err = json.Unmarshal([]byte(line[4]), &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
videos = append(videos, sources.NewUCBVideo(line[0], line[2], line[1], line[3], data.PublishedAt, s.videoDirectory))
|
||||
}
|
||||
|
||||
log.Printf("Publishing %d videos\n", len(videos))
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
break Enqueue
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case s.queue <- v:
|
||||
case <-s.grp.Ch():
|
||||
break Enqueue
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
@@ -823,6 +793,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
sv, ok := s.syncedVideos[v.ID()]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
alreadyPublished := ok && sv.Published
|
||||
isUpgradeSync := s.Manager.syncStatus == StatusPendingUpgrade
|
||||
|
||||
neverRetryFailures := []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
@@ -838,10 +809,15 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if alreadyPublished {
|
||||
if alreadyPublished && !isUpgradeSync {
|
||||
log.Println(v.ID() + " already published")
|
||||
return nil
|
||||
}
|
||||
newMetadataVersion := int8(2)
|
||||
if ok && sv.MetadataVersion >= newMetadataVersion {
|
||||
log.Println(v.ID() + " upgraded to the new metadata")
|
||||
return nil
|
||||
}
|
||||
|
||||
if v.PlaylistPosition() > s.Manager.videosLimit {
|
||||
log.Println(v.ID() + " is old: skipping")
|
||||
@@ -851,8 +827,16 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sp := sources.SyncParams{
|
||||
ClaimAddress: s.claimAddress,
|
||||
Amount: publishAmount,
|
||||
ChannelID: s.lbryChannelID,
|
||||
MaxVideoSize: s.Manager.maxVideoSize,
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.lbryChannelID, s.Manager.maxVideoSize, s.namer, s.Manager.maxVideoLength)
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+11
-5
@@ -76,10 +76,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
}
|
||||
|
||||
type SyncedVideo struct {
|
||||
VideoID string `json:"video_id"`
|
||||
Published bool `json:"published"`
|
||||
FailureReason string `json:"failure_reason"`
|
||||
ClaimName string `json:"claim_name"`
|
||||
VideoID string `json:"video_id"`
|
||||
Published bool `json:"published"`
|
||||
FailureReason string `json:"failure_reason"`
|
||||
ClaimName string `json:"claim_name"`
|
||||
ClaimID string `json:"claim_id"`
|
||||
Size int64 `json:"size"`
|
||||
MetadataVersion int8 `json:"metadata_version"`
|
||||
}
|
||||
|
||||
func sanitizeFailureReason(s *string) {
|
||||
@@ -121,7 +124,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
claimNames := make(map[string]bool)
|
||||
for _, v := range response.Data {
|
||||
svs[v.VideoID] = v
|
||||
claimNames[v.ClaimName] = v.Published
|
||||
if v.ClaimName != "" {
|
||||
claimNames[v.ClaimName] = v.Published
|
||||
}
|
||||
}
|
||||
return svs, claimNames, nil
|
||||
}
|
||||
@@ -178,6 +183,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
|
||||
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
vals.Add("claim_id", claimID)
|
||||
vals.Add("claim_name", claimName)
|
||||
vals.Add("metadata_version", "2")
|
||||
if size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*size, 10))
|
||||
}
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type ucbVideo struct {
|
||||
id string
|
||||
title string
|
||||
channel string
|
||||
description string
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
claimNames map[string]bool
|
||||
syncedVideosMux *sync.RWMutex
|
||||
}
|
||||
|
||||
func NewUCBVideo(id, title, channel, description, publishedAt, dir string) *ucbVideo {
|
||||
p, _ := time.Parse(time.RFC3339Nano, publishedAt) // ignore parse errors
|
||||
return &ucbVideo{
|
||||
id: id,
|
||||
title: title,
|
||||
description: description,
|
||||
channel: channel,
|
||||
dir: dir,
|
||||
publishedAt: p,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *ucbVideo) ID() string {
|
||||
return v.id
|
||||
}
|
||||
|
||||
func (v *ucbVideo) PlaylistPosition() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (v *ucbVideo) IDAndNum() string {
|
||||
return v.ID() + " (?)"
|
||||
}
|
||||
|
||||
func (v *ucbVideo) PublishedAt() time.Time {
|
||||
return v.publishedAt
|
||||
//r := regexp.MustCompile(`(\d\d\d\d)-(\d\d)-(\d\d)`)
|
||||
//matches := r.FindStringSubmatch(v.title)
|
||||
//if len(matches) > 0 {
|
||||
// year, _ := strconv.Atoi(matches[1])
|
||||
// month, _ := strconv.Atoi(matches[2])
|
||||
// day, _ := strconv.Atoi(matches[3])
|
||||
// return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
|
||||
//}
|
||||
//return time.Now()
|
||||
}
|
||||
|
||||
func (v *ucbVideo) getFilename() string {
|
||||
return v.dir + "/" + v.id + ".mp4"
|
||||
}
|
||||
|
||||
func (v *ucbVideo) getClaimName(attempt int) string {
|
||||
reg := regexp.MustCompile(`[^a-zA-Z0-9]+`)
|
||||
suffix := ""
|
||||
if attempt > 1 {
|
||||
suffix = "-" + strconv.Itoa(attempt)
|
||||
}
|
||||
maxLen := 40 - len(suffix)
|
||||
|
||||
chunks := strings.Split(strings.ToLower(strings.Trim(reg.ReplaceAllString(v.title, "-"), "-")), "-")
|
||||
|
||||
name := chunks[0]
|
||||
if len(name) > maxLen {
|
||||
return name[:maxLen]
|
||||
}
|
||||
|
||||
for _, chunk := range chunks[1:] {
|
||||
tmpName := name + "-" + chunk
|
||||
if len(tmpName) > maxLen {
|
||||
if len(name) < 20 {
|
||||
name = tmpName[:maxLen]
|
||||
}
|
||||
break
|
||||
}
|
||||
name = tmpName
|
||||
}
|
||||
|
||||
return name + suffix
|
||||
}
|
||||
|
||||
func (v *ucbVideo) getAbbrevDescription() string {
|
||||
maxLines := 10
|
||||
description := strings.TrimSpace(v.description)
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..."
|
||||
}
|
||||
|
||||
func (v *ucbVideo) download() error {
|
||||
videoPath := v.getFilename()
|
||||
|
||||
_, err := os.Stat(videoPath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
} else if err == nil {
|
||||
log.Debugln(v.id + " already exists at " + videoPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "")
|
||||
s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s)
|
||||
|
||||
out, err := os.Create(videoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
log.Println("lbry-niko2/videos/" + v.channel + "/" + v.id)
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String("lbry-niko2"),
|
||||
Key: aws.String("/videos/" + v.channel + "/" + v.id + ".mp4"),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *ucbVideo) saveThumbnail() error {
|
||||
resp, err := http.Get("https://s3.us-east-2.amazonaws.com/lbry-niko2/thumbnails/" + v.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "")
|
||||
s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
uploader := s3manager.NewUploader(s)
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String("berk.ninja"),
|
||||
Key: aws.String("thumbnails/" + v.id),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
Body: resp.Body,
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: v.title,
|
||||
Description: v.getAbbrevDescription(),
|
||||
ClaimAddress: &claimAddress,
|
||||
Languages: []string{"en"},
|
||||
ThumbnailURL: strPtr("https://berk.ninja/thumbnails/" + v.id),
|
||||
Tags: []string{},
|
||||
},
|
||||
Author: strPtr("UC Berkeley"),
|
||||
License: strPtr("see description"),
|
||||
StreamType: &jsonrpc.StreamTypeVideo,
|
||||
ChannelID: &channelID,
|
||||
}
|
||||
return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer)
|
||||
}
|
||||
|
||||
func (v *ucbVideo) Size() *int64 {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) {
|
||||
//download and thumbnail can be done in parallel
|
||||
err := v.download()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
//err = v.SaveThumbnail()
|
||||
//if err != nil {
|
||||
// return errors.WrapPrefix(err, "thumbnail error", 0)
|
||||
//}
|
||||
//log.Debugln("Created thumbnail for " + v.id)
|
||||
|
||||
summary, err := v.publish(daemon, claimAddress, amount, channelID, namer)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("publish error", err)
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
+228
-77
@@ -1,12 +1,12 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -15,9 +15,14 @@ import (
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/nikooo777/ytdl"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
@@ -36,9 +41,49 @@ type YoutubeVideo struct {
|
||||
dir string
|
||||
youtubeInfo *youtube.Video
|
||||
tags []string
|
||||
awsConfig aws.Config
|
||||
thumbnailURL string
|
||||
lbryChannelID string
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64) *YoutubeVideo {
|
||||
const reflectorURL = "http://blobs.lbry.io/"
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
"1": "Film & Animation",
|
||||
"2": "Autos & Vehicles",
|
||||
"10": "Music",
|
||||
"15": "Pets & Animals",
|
||||
"17": "Sports",
|
||||
"18": "Short Movies",
|
||||
"19": "Travel & Events",
|
||||
"20": "Gaming",
|
||||
"21": "Videoblogging",
|
||||
"22": "People & Blogs",
|
||||
"23": "Comedy",
|
||||
"24": "Entertainment",
|
||||
"25": "News & Politics",
|
||||
"26": "Howto & Style",
|
||||
"27": "Education",
|
||||
"28": "Science & Technology",
|
||||
"29": "Nonprofits & Activism",
|
||||
"30": "Movies",
|
||||
"31": "Anime/Animation",
|
||||
"32": "Action/Adventure",
|
||||
"33": "Classics",
|
||||
"34": "Comedy",
|
||||
"35": "Documentary",
|
||||
"36": "Drama",
|
||||
"37": "Family",
|
||||
"38": "Foreign",
|
||||
"39": "Horror",
|
||||
"40": "Sci-Fi/Fantasy",
|
||||
"41": "Thriller",
|
||||
"42": "Shorts",
|
||||
"43": "Shows",
|
||||
"44": "Trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
@@ -49,6 +94,7 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
|
||||
publishedAt: publishedAt,
|
||||
dir: directory,
|
||||
youtubeInfo: videoData,
|
||||
awsConfig: awsConfig,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,9 +147,30 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description
|
||||
}
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
if v.lbryChannelID == khanAcademyClaimID {
|
||||
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..."
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) fallbackDownload() error {
|
||||
cmd := exec.Command("youtube-dl",
|
||||
v.ID(),
|
||||
"--no-progress",
|
||||
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+bestaudio/best[ext=mp4,height<=1080,filesize<2000M]",
|
||||
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"))
|
||||
log.Printf("Running command and waiting for it to finish...")
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.Debugln(string(output))
|
||||
if err != nil {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(output)))
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
@@ -190,108 +257,108 @@ func (v *YoutubeVideo) download() error {
|
||||
func (v *YoutubeVideo) videoDir() string {
|
||||
return v.dir + "/" + v.id
|
||||
}
|
||||
func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
files, err := ioutil.ReadDir(v.videoDir())
|
||||
log.Infoln(v.videoDir())
|
||||
if err != nil {
|
||||
err = errors.Prefix("list error", err)
|
||||
log.Errorln(err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
|
||||
return v.videoDir() + "/" + f.Name(), nil
|
||||
}
|
||||
}
|
||||
return "", errors.Err("could not find any downloaded videos")
|
||||
|
||||
}
|
||||
func (v *YoutubeVideo) delete() error {
|
||||
videoPath := v.getFullPath()
|
||||
err := os.Remove(videoPath)
|
||||
videoPath, err := v.getDownloadedPath()
|
||||
if err != nil {
|
||||
log.Errorln(errors.Prefix("delete error", err))
|
||||
log.Errorln(err)
|
||||
return err
|
||||
}
|
||||
log.Debugln(v.id + " deleted from disk (" + videoPath + ")")
|
||||
return nil
|
||||
}
|
||||
err = os.Remove(videoPath)
|
||||
log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() error {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
params, err := json.Marshal(map[string]string{"videoid": v.id})
|
||||
if err != nil {
|
||||
err = errors.Prefix("delete error", err)
|
||||
log.Errorln(err)
|
||||
return err
|
||||
}
|
||||
|
||||
request, err := http.NewRequest(http.MethodPut, "https://jgp4g1qoud.execute-api.us-east-1.amazonaws.com/prod/thumbnail", bytes.NewBuffer(params))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
contents, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Error int `json:"error"`
|
||||
Url string `json:"url,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
err = json.Unmarshal(contents, &decoded)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if decoded.Error != 0 {
|
||||
return errors.Err("error creating thumbnail: " + decoded.Message)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
if channelID == khanAcademyClaimID {
|
||||
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
|
||||
}
|
||||
var languages []string = nil
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, namer *namer.Namer) (*SyncSummary, error) {
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: v.title,
|
||||
Description: v.getAbbrevDescription() + additionalDescription,
|
||||
Description: v.getAbbrevDescription(),
|
||||
ClaimAddress: &claimAddress,
|
||||
Languages: languages,
|
||||
ThumbnailURL: strPtr("https://thumbnails.lbry.com/" + v.id),
|
||||
Tags: v.youtubeInfo.Snippet.Tags,
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
},
|
||||
Author: strPtr(v.channelTitle),
|
||||
License: strPtr("Copyrighted (contact author)"),
|
||||
StreamType: &jsonrpc.StreamTypeVideo,
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
|
||||
ChannelID: &channelID,
|
||||
Author: util.PtrToString(v.channelTitle),
|
||||
License: util.PtrToString("Copyrighted (contact author)"),
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
}
|
||||
|
||||
return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
|
||||
downloadPath, err := v.getDownloadedPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return publishAndRetryExistingNames(daemon, v.title, downloadPath, amount, options, namer)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Size() *int64 {
|
||||
return v.size
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(maxVideoSize) * 1024 * 1024
|
||||
v.maxVideoLength = maxVideoLength
|
||||
//download and thumbnail can be done in parallel
|
||||
type SyncParams struct {
|
||||
ClaimAddress string
|
||||
Amount float64
|
||||
ChannelID string
|
||||
MaxVideoSize int
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
|
||||
v.maxVideoLength = params.MaxVideoLength
|
||||
v.lbryChannelID = params.ChannelID
|
||||
if reprocess && existingVideoData != nil && existingVideoData.Published {
|
||||
summary, err := v.reprocess(daemon, params, existingVideoData)
|
||||
return summary, err
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
|
||||
fallBackErr := v.fallbackDownload()
|
||||
if fallBackErr != nil {
|
||||
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
|
||||
return nil, errors.Prefix("download error", err) //return original error
|
||||
}
|
||||
}
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
@@ -301,9 +368,93 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount
|
||||
}
|
||||
log.Debugln("Created thumbnail for " + v.id)
|
||||
|
||||
summary, err := v.publish(daemon, claimAddress, amount, channelID, namer)
|
||||
summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.Namer)
|
||||
//delete the video in all cases (and ignore the error)
|
||||
_ = v.delete()
|
||||
|
||||
return summary, errors.Prefix("publish error", err)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Location, tags []string) {
|
||||
languages = nil
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
|
||||
locations = nil
|
||||
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
locations = []jsonrpc.Location{{
|
||||
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
|
||||
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
|
||||
}}
|
||||
}
|
||||
tags = append([]string{youtubeCategories[v.youtubeInfo.Snippet.CategoryId]}, v.youtubeInfo.Snippet.Tags...)
|
||||
tags = tagsManager.SanitizeTags(tags)
|
||||
return languages, locations, tags
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if len(c.Claims) == 0 {
|
||||
return nil, errors.Err("cannot reprocess: no claim found for this video")
|
||||
} else if len(c.Claims) > 1 {
|
||||
return nil, errors.Err("cannot reprocess: too many claims. claimID: %s", existingVideoData.ClaimID)
|
||||
}
|
||||
|
||||
currentClaim := c.Claims[0]
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
thumbnailURL := ""
|
||||
if currentClaim.Value.GetThumbnail() == nil {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
} else {
|
||||
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
|
||||
}
|
||||
|
||||
videoSize, err := currentClaim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
if existingVideoData.Size > 0 {
|
||||
videoSize = uint64(existingVideoData.Size)
|
||||
} else {
|
||||
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
|
||||
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
}
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: v.title,
|
||||
Description: v.getAbbrevDescription(),
|
||||
Tags: tags,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact author)"),
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
},
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SyncSummary{
|
||||
ClaimID: pr.Outputs[0].ClaimID,
|
||||
ClaimName: pr.Outputs[0].Name,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,818 @@
|
||||
package tagsManager
|
||||
|
||||
const (
|
||||
Gaming = "gaming"
|
||||
Blockchain = "blockchain"
|
||||
News = "news"
|
||||
Learning = "learning"
|
||||
Funny = "funny"
|
||||
Technology = "technology"
|
||||
Automotive = "automotive"
|
||||
Economics = "economics"
|
||||
Sports = "sports"
|
||||
Food = "food"
|
||||
Science = "science"
|
||||
Fitness = "fitness"
|
||||
Art = "art"
|
||||
Nature = "nature"
|
||||
)
|
||||
|
||||
func SanitizeTags(tags []string) []string {
|
||||
ts := &tagsSanitizer{
|
||||
Unsanitized: tags,
|
||||
}
|
||||
ts.init()
|
||||
ts.cleanup()
|
||||
ts.replace()
|
||||
ts.add()
|
||||
|
||||
sanitizedTags := make([]string, len(ts.Sanitized))
|
||||
index := 0
|
||||
for t := range ts.Sanitized {
|
||||
sanitizedTags[index] = t
|
||||
index++
|
||||
}
|
||||
return sanitizedTags
|
||||
}
|
||||
|
||||
type tagsSanitizer struct {
|
||||
Unsanitized []string
|
||||
Sanitized map[string]bool
|
||||
}
|
||||
|
||||
func (ts *tagsSanitizer) init() {
|
||||
if len(ts.Sanitized) == 0 {
|
||||
ts.Sanitized = make(map[string]bool, len(ts.Unsanitized)+14)
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *tagsSanitizer) cleanup() {
|
||||
for _, t := range ts.Unsanitized {
|
||||
_, ok := tagsToSkip[t]
|
||||
if !ok {
|
||||
ts.Sanitized[t] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *tagsSanitizer) replace() {
|
||||
for _, t := range ts.Unsanitized {
|
||||
match, filterMatch := mapAndReplace[t]
|
||||
if filterMatch {
|
||||
delete(ts.Sanitized, t)
|
||||
ts.Sanitized[match] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *tagsSanitizer) add() {
|
||||
for _, t := range ts.Unsanitized {
|
||||
match, filterMatch := mapAndKeep[t]
|
||||
if filterMatch {
|
||||
ts.Sanitized[match] = true
|
||||
ts.Sanitized[t] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var tagsToSkip = map[string]*struct{}{
|
||||
"video": nil,
|
||||
"review": nil,
|
||||
"youtube": nil,
|
||||
"david packman": nil,
|
||||
"2018": nil,
|
||||
"new": nil,
|
||||
"the": nil,
|
||||
"music": nil,
|
||||
"hd": nil,
|
||||
"talk": nil,
|
||||
"entertainment": nil,
|
||||
"Radio": nil,
|
||||
"2017": nil,
|
||||
"live": nil,
|
||||
"free": nil,
|
||||
"america": nil,
|
||||
"of": nil,
|
||||
"david pakman": nil,
|
||||
"dave pacman": nil,
|
||||
"pakman": nil,
|
||||
"davidpakman.com": nil,
|
||||
"tdps": nil,
|
||||
"1080p": nil,
|
||||
"united": nil,
|
||||
"david pakman show": nil,
|
||||
"states": nil,
|
||||
"europe": nil,
|
||||
"tomonews": nil,
|
||||
"Tomonews": nil,
|
||||
"2016": nil,
|
||||
"eeuu": nil,
|
||||
"REVIEW": nil,
|
||||
"Launch": nil,
|
||||
"taiwanese animators": nil,
|
||||
"best": nil,
|
||||
"TURBO": nil,
|
||||
"television": nil,
|
||||
"gt": nil,
|
||||
"USA": nil,
|
||||
"GPS": nil,
|
||||
"PREMIERE": nil,
|
||||
"guide": nil,
|
||||
"scoope": nil,
|
||||
"US": nil,
|
||||
"nma": nil,
|
||||
"Nma": nil,
|
||||
"to": nil,
|
||||
"auto imagen": nil,
|
||||
"tips": nil,
|
||||
"action": nil,
|
||||
"download": nil,
|
||||
"david pacman": nil,
|
||||
"taiwanese animation": nil,
|
||||
"Taiwanese animation": nil,
|
||||
"YouTube Editor": nil,
|
||||
"Tomo news": nil,
|
||||
"replay": nil,
|
||||
"update": nil,
|
||||
"battle": nil,
|
||||
"2019": nil,
|
||||
"story": nil,
|
||||
"one": nil,
|
||||
"spreaker": nil,
|
||||
"Jay's": nil,
|
||||
"episode": nil,
|
||||
"for": nil,
|
||||
"SHAREfactory™": nil,
|
||||
"#hangoutsonair": nil,
|
||||
"#hoa": nil,
|
||||
"vs": nil,
|
||||
"bob": nil,
|
||||
"dnb portal": nil,
|
||||
"instagram": nil,
|
||||
"challenge": nil,
|
||||
"eatmydiction1": nil,
|
||||
"usa": nil,
|
||||
"français": nil,
|
||||
"live stream": nil,
|
||||
"top": nil,
|
||||
"fr": nil,
|
||||
"fanta": nil,
|
||||
"HD": nil,
|
||||
"and": nil,
|
||||
"foto": nil,
|
||||
"no": nil,
|
||||
"720p": nil,
|
||||
"up": nil,
|
||||
"radio": nil,
|
||||
"david": nil,
|
||||
"russia": nil,
|
||||
"ytquality=high": nil,
|
||||
"part": nil,
|
||||
"bollywood tashan": nil,
|
||||
"bounty": nil,
|
||||
"dnb": nil,
|
||||
"highlights": nil,
|
||||
"thefantasio974": nil,
|
||||
"great": nil,
|
||||
"man": nil,
|
||||
"replica": nil,
|
||||
"red": nil,
|
||||
"chiara ferragni": nil,
|
||||
"in": nil,
|
||||
"bob lennon": nil,
|
||||
"series": nil,
|
||||
"life": nil,
|
||||
"first": nil,
|
||||
"dnbportal": nil,
|
||||
"fantabobgames": nil,
|
||||
"segui": nil,
|
||||
"lennon": nil,
|
||||
"neurofunk": nil,
|
||||
"on": nil,
|
||||
"kag3": nil,
|
||||
"2015": nil,
|
||||
"ITA": nil,
|
||||
"gamerworf": nil,
|
||||
"chickens": nil,
|
||||
"uct-wqktykk1_70u4bb4k4lq": nil,
|
||||
"beard": nil,
|
||||
"videos": nil,
|
||||
"nidge": nil,
|
||||
"bill still": nil,
|
||||
"nickatnyte": nil,
|
||||
"прохождение": nil,
|
||||
"minimal": nil,
|
||||
"clash": nil,
|
||||
"trofeo": nil,
|
||||
"attack": nil,
|
||||
"test": nil,
|
||||
"DFX": nil,
|
||||
"india": nil,
|
||||
"police": nil,
|
||||
"campaign": nil,
|
||||
"tricks": nil,
|
||||
"depression": nil,
|
||||
"unique": nil,
|
||||
"Bharat": nil,
|
||||
"imagen": nil,
|
||||
"LanceScurv": nil,
|
||||
"us": nil,
|
||||
"google": nil,
|
||||
"defense": nil,
|
||||
"gameplay fr": nil,
|
||||
"giveaway": nil,
|
||||
"hot": nil,
|
||||
"Interior": nil,
|
||||
"show": nil,
|
||||
"YouTube Capture": nil,
|
||||
"jump": nil,
|
||||
"2014": nil,
|
||||
"thailand": nil,
|
||||
"santa": nil,
|
||||
"ultimate": nil,
|
||||
"deutsch": nil,
|
||||
"mission": nil,
|
||||
"The Lancescurv Show": nil,
|
||||
"Legends": nil,
|
||||
"modi": nil,
|
||||
"noise": nil,
|
||||
"Texas": nil,
|
||||
"china": nil,
|
||||
"Scurv": nil,
|
||||
"gift": nil,
|
||||
"sikh": nil,
|
||||
"night": nil,
|
||||
"Episode": nil,
|
||||
"good": nil,
|
||||
"plus": nil,
|
||||
"szekelyvegan": nil,
|
||||
"székelyvegán": nil,
|
||||
"szekely": nil,
|
||||
"игра": nil,
|
||||
"is": nil,
|
||||
"székely vegán": nil,
|
||||
"john sonmez": nil,
|
||||
"jew": nil,
|
||||
"magyar vlogger": nil,
|
||||
"magyar vlog": nil,
|
||||
"hungarian vlog": nil,
|
||||
"baba": nil,
|
||||
"montage": nil,
|
||||
"New": nil,
|
||||
"squad": nil,
|
||||
"german": nil,
|
||||
"amerika": nil,
|
||||
"full": nil,
|
||||
"sri": nil,
|
||||
"kristomaster4": nil,
|
||||
"narendra": nil,
|
||||
"sant": nil,
|
||||
"adityanath": nil,
|
||||
"sadhguru": nil,
|
||||
"ped": nil,
|
||||
"Sadhu": nil,
|
||||
"Shyam": nil,
|
||||
"dharm": nil,
|
||||
"babas": nil,
|
||||
"gaay": nil,
|
||||
"Pradesh": nil,
|
||||
"Raebareli": nil,
|
||||
"waale": nil,
|
||||
"Uttar": nil,
|
||||
"Bandar": nil,
|
||||
"kutta": nil,
|
||||
"bakri": nil,
|
||||
"aughad": nil,
|
||||
"suar": nil,
|
||||
"jaanwar": nil,
|
||||
"tona": nil,
|
||||
"bhawreshwara": nil,
|
||||
"vaanar": nil,
|
||||
"bhagwa": nil,
|
||||
"prakriti": nil,
|
||||
"samaj": nil,
|
||||
"totka": nil,
|
||||
"bhoj": nil,
|
||||
"dave": nil,
|
||||
"vulcanhdgaming": nil,
|
||||
"fast": nil,
|
||||
"moon": nil,
|
||||
"scene": nil,
|
||||
"tv": nil,
|
||||
"2": nil,
|
||||
"fighter": nil,
|
||||
"NEW": nil,
|
||||
"jeux": nil,
|
||||
"игры": nil,
|
||||
"tanyázás": nil,
|
||||
"obiettivo": nil,
|
||||
"csgo": nil,
|
||||
"The guy From Pittsburgh": nil,
|
||||
"blue": nil,
|
||||
"youtuber": nil,
|
||||
"clash royale": nil,
|
||||
"SHOW": nil,
|
||||
"johnsp69": nil,
|
||||
"japan": nil,
|
||||
"YouTube": nil,
|
||||
"The David Pakman Show": nil,
|
||||
"thecreativeone": nil,
|
||||
"Youtube": nil,
|
||||
"classic": nil,
|
||||
"Talk Radio": nil,
|
||||
"white": nil,
|
||||
"летсплей": nil,
|
||||
"clash of clans": nil,
|
||||
"Industry (Organization Sector)": nil,
|
||||
"difranco": nil,
|
||||
"custom": nil,
|
||||
"прохождение игры": nil,
|
||||
"progressive podcast": nil,
|
||||
"george Senda": nil,
|
||||
"360": nil,
|
||||
"betv": nil,
|
||||
"gamingoncaffeine": nil,
|
||||
"Drive": nil,
|
||||
"Africa": nil,
|
||||
"progressive news": nil,
|
||||
"full hd": nil,
|
||||
"Asia": nil,
|
||||
"liberal news": nil,
|
||||
"packman": nil,
|
||||
"hdtv": nil,
|
||||
"Entertainment": nil,
|
||||
"Simple Programmer": nil,
|
||||
"indonesia": nil,
|
||||
"karmakut": nil,
|
||||
"gorthemoviegod": nil,
|
||||
"Lance scurvin": nil,
|
||||
"silver": nil,
|
||||
"clans": nil,
|
||||
"игры для мальчиков": nil,
|
||||
"игры для девочек": nil,
|
||||
"blackpeace72": nil,
|
||||
"multicolored": nil,
|
||||
"intro": nil,
|
||||
"creative beard": nil,
|
||||
"cyber locks": nil,
|
||||
"uniqornaments": nil,
|
||||
"daily holidays": nil,
|
||||
"daily celebration": nil,
|
||||
"holiday everyday": nil,
|
||||
"3d": nil,
|
||||
"lp": nil,
|
||||
"interview": nil,
|
||||
"Heroes": nil,
|
||||
"maskedmage": nil,
|
||||
"рпг": nil,
|
||||
"north": nil,
|
||||
"road": nil,
|
||||
"monster": nil,
|
||||
"de": nil,
|
||||
"mode": nil,
|
||||
"podcastradio": nil,
|
||||
"Man": nil,
|
||||
"preview": nil,
|
||||
"rage": nil,
|
||||
"map": nil,
|
||||
"Pittsburgh pa": nil,
|
||||
"david di franco": nil,
|
||||
"win": nil,
|
||||
"junior senior": nil,
|
||||
"demo": nil,
|
||||
"paul": nil,
|
||||
"simpleprogrammer.com": nil,
|
||||
"2013": nil,
|
||||
"green": nil,
|
||||
"la": nil,
|
||||
"london": nil,
|
||||
"adam": nil,
|
||||
"jnrsnrgaming": nil,
|
||||
"jnrsnr": nil,
|
||||
"TRUE": nil,
|
||||
"junior senior gaming": nil,
|
||||
"the kag3 gaming": nil,
|
||||
"sonny daniel": nil,
|
||||
"atlanta": nil,
|
||||
"wordofgod": nil,
|
||||
"2012": nil,
|
||||
"Katrina kaif": nil,
|
||||
"English": nil,
|
||||
"Addon": nil,
|
||||
"High": nil,
|
||||
"To": nil,
|
||||
"willie": nil,
|
||||
"it": nil,
|
||||
"trolling": nil,
|
||||
"Bollywood news": nil,
|
||||
"willie pelissier": nil,
|
||||
"canada": nil,
|
||||
"productions": nil,
|
||||
"iMovie": nil,
|
||||
"channel": nil,
|
||||
"total": nil,
|
||||
"trick": nil,
|
||||
"the kag3": nil,
|
||||
"Scurvin": nil,
|
||||
"Free": nil,
|
||||
"make": nil,
|
||||
"The": nil,
|
||||
"ops": nil,
|
||||
"trevor": nil,
|
||||
"kevin": nil,
|
||||
"life is strange": nil,
|
||||
"you": nil,
|
||||
"kokesh": nil,
|
||||
"the brotherhood of gaming": nil,
|
||||
"Martinez ca": nil,
|
||||
"real": nil,
|
||||
"เกม": nil,
|
||||
"sonny daniel vlogs": nil,
|
||||
"China": nil,
|
||||
"African American": nil,
|
||||
"Slime": nil,
|
||||
"America": nil,
|
||||
"price": nil,
|
||||
"michigan": nil,
|
||||
"call": nil,
|
||||
"truetotalempireinc": nil,
|
||||
"Akshay Kumar": nil,
|
||||
"Shield": nil,
|
||||
"aventure": nil,
|
||||
"until dawn": nil,
|
||||
"camera": nil,
|
||||
"florida": nil,
|
||||
"with": nil,
|
||||
"battlefield": nil,
|
||||
"joe": nil,
|
||||
"tom": nil,
|
||||
"Atmospheric": nil,
|
||||
"squad ops": nil,
|
||||
"overview": nil,
|
||||
"fist": nil,
|
||||
"clan": nil,
|
||||
"guy": nil,
|
||||
"sonnydaniel": nil,
|
||||
"Voice Over": nil,
|
||||
"Test": nil,
|
||||
"HouseParty": nil,
|
||||
"ending": nil,
|
||||
"my": nil,
|
||||
"garena": nil,
|
||||
"kag3 entertainment": nil,
|
||||
"{5859dfec-026f-46ba-bea0-02bf43aa1a6f}": nil,
|
||||
"shahrukh Khan": nil,
|
||||
"Opening": nil,
|
||||
"erik": nil,
|
||||
"part 1": nil,
|
||||
"UK": nil,
|
||||
"dean": nil,
|
||||
"hcg": nil,
|
||||
"4k": nil,
|
||||
"cheat": nil,
|
||||
}
|
||||
var mapAndReplace = map[string]string{
|
||||
"nfl": Sports,
|
||||
"Minecraft (Award-Winning Work)": Gaming,
|
||||
"minecraft survival": Gaming,
|
||||
"modded minecraft": Gaming,
|
||||
"minecraft movie": Gaming,
|
||||
"minecraft videos": Gaming,
|
||||
}
|
||||
|
||||
var mapAndKeep = map[string]string{
|
||||
"gameplay": Gaming,
|
||||
"games": Gaming,
|
||||
"News": News,
|
||||
"game": Gaming,
|
||||
"politics": News,
|
||||
"PlayStation 4": Gaming,
|
||||
"Sony Interactive Entertainment": Gaming,
|
||||
"walkthrough": Gaming,
|
||||
"pc": Technology,
|
||||
"playthrough": Gaming,
|
||||
"#PS4Live": Gaming,
|
||||
"let's play": Gaming,
|
||||
"democrat": News,
|
||||
"liberal": News,
|
||||
"progressive": News,
|
||||
"government": News,
|
||||
"play": Gaming,
|
||||
"republican": News,
|
||||
"conservative": News,
|
||||
"commentary": News,
|
||||
"political": News,
|
||||
"CNN": News,
|
||||
"tutorial": Learning,
|
||||
"cbs": News,
|
||||
"lets play": Gaming,
|
||||
"MSNBC": News,
|
||||
"bitcoin": Blockchain,
|
||||
"minecraft": Gaming,
|
||||
"twitch": Gaming,
|
||||
"Video Game (Industry)": Gaming,
|
||||
"ps4": Gaming,
|
||||
"#PS4share": Gaming,
|
||||
"how to": Learning,
|
||||
"Fox news": News,
|
||||
"fox": News,
|
||||
"xbox": Gaming,
|
||||
"nbc": News,
|
||||
"Gaming": Gaming,
|
||||
"comedy": Funny,
|
||||
"video game": Gaming,
|
||||
"car": Automotive,
|
||||
"Education": Learning,
|
||||
"POLITICS": News,
|
||||
"Educational": Learning,
|
||||
"cryptocurrency": Blockchain,
|
||||
"News & Politics": News,
|
||||
"fun": Funny,
|
||||
"playstation": Gaming,
|
||||
"CARS": Automotive,
|
||||
"xbox one": Gaming,
|
||||
"senate": News,
|
||||
"steam": Gaming,
|
||||
"4x4": Automotive,
|
||||
"AUTOMOBILE": Automotive,
|
||||
"fortnite": Gaming,
|
||||
"autos": Automotive,
|
||||
"Sport": Sports,
|
||||
"crypto": Blockchain,
|
||||
"Carros": Automotive,
|
||||
"ethereum": Blockchain,
|
||||
"Suv": Automotive,
|
||||
"playstation 4": Gaming,
|
||||
"TRUCK": Automotive,
|
||||
"video games": Gaming,
|
||||
"Funny": Funny,
|
||||
"Garage": Automotive,
|
||||
"congress": News,
|
||||
"lets": Gaming,
|
||||
"nintendo": Gaming,
|
||||
"ps3": Gaming,
|
||||
"auto": Automotive,
|
||||
"house of representatives": News,
|
||||
"fps": Gaming,
|
||||
"rpg": Gaming,
|
||||
"btc": Blockchain,
|
||||
"video game (industry)": Gaming,
|
||||
"multiplayer": Gaming,
|
||||
"video news": News,
|
||||
"animated news": News,
|
||||
"android": Technology,
|
||||
"computer": Technology,
|
||||
"how": Learning,
|
||||
"let's": Gaming,
|
||||
"donald trump": News,
|
||||
"Fortnite": Gaming,
|
||||
"COMO": News,
|
||||
"online": Technology,
|
||||
"TRUMP": News,
|
||||
"mods": Technology,
|
||||
"tech": Technology,
|
||||
"hilarious": Funny,
|
||||
"dota 2": Gaming,
|
||||
"crash": Automotive,
|
||||
"PUTIN": News,
|
||||
"dance": Art,
|
||||
"money": Economics,
|
||||
"news radio": News,
|
||||
"lol": Funny,
|
||||
"xbox 360": Gaming,
|
||||
"SALMAN KHAN": Learning,
|
||||
"airdrop": Blockchain,
|
||||
"litecoin": Blockchain,
|
||||
"Sony Computer Entertainment": Gaming,
|
||||
"pc games": Gaming,
|
||||
"eth": Blockchain,
|
||||
"mining": Blockchain,
|
||||
"ios": Technology,
|
||||
"linux": Technology,
|
||||
"sony": Technology,
|
||||
"space": Nature,
|
||||
"iphone": Technology,
|
||||
"education": Learning,
|
||||
"Gamer": Gaming,
|
||||
"bitcoin news": Blockchain,
|
||||
"call of duty": Gaming,
|
||||
"pokemon": Gaming,
|
||||
"League": Gaming,
|
||||
"Let's Play": Gaming,
|
||||
"Let's": Gaming,
|
||||
"league of legends": Gaming,
|
||||
"League Of Legends (Video Game)": Gaming,
|
||||
"economy": Economics,
|
||||
"cod": Gaming,
|
||||
"ico": Blockchain,
|
||||
"pubg": Gaming,
|
||||
"gold": Economics,
|
||||
"Xbox": Gaming,
|
||||
"pvp": Gaming,
|
||||
"videogame": Gaming,
|
||||
"vegan": Food,
|
||||
"dota": Gaming,
|
||||
"Games": Gaming,
|
||||
"let's play fr": Gaming,
|
||||
"satire": Funny,
|
||||
"mario": Gaming,
|
||||
"pc game": Gaming,
|
||||
"help": Learning,
|
||||
"PS4": Gaming,
|
||||
"crypto news": Blockchain,
|
||||
"humor": Funny,
|
||||
"let": Gaming,
|
||||
"fallout 4": Gaming,
|
||||
"Shooter Game (Media Genre)": Gaming,
|
||||
"funny moments": Funny,
|
||||
"Comedy": Funny,
|
||||
"Playthrough": Gaming,
|
||||
"gamer": Gaming,
|
||||
"wii": Gaming,
|
||||
"playing": Gaming,
|
||||
"PC": Technology,
|
||||
"computer game games": Gaming,
|
||||
"Gameplay": Gaming,
|
||||
"arcade": Gaming,
|
||||
"token": Blockchain,
|
||||
"federal reserve": Economics,
|
||||
"Video Game Culture": Gaming,
|
||||
"driving": Automotive,
|
||||
"plants": Nature,
|
||||
"trees": Nature,
|
||||
"eggs": Nature,
|
||||
"Xbox 360 (Video Game Platform)": Gaming,
|
||||
"RPG": Gaming,
|
||||
"design": Art,
|
||||
"market": Economics,
|
||||
"cars": Automotive,
|
||||
"fruits": Food,
|
||||
"flowers": Nature,
|
||||
"ram": Technology,
|
||||
"lessons": Learning,
|
||||
"token free": Blockchain,
|
||||
"vegán": Food,
|
||||
"Game": Gaming,
|
||||
"investing": Economics,
|
||||
"samsung": Technology,
|
||||
"racing": Sports,
|
||||
"online learning": Learning,
|
||||
"dollar": Economics,
|
||||
"software": Technology,
|
||||
"beach": Nature,
|
||||
"gta": Gaming,
|
||||
"windows": Technology,
|
||||
"let’s play": Gaming,
|
||||
"Minecraft": Gaming,
|
||||
"race": Automotive,
|
||||
"skyrim": Gaming,
|
||||
"rainbow": Nature,
|
||||
"health": Fitness,
|
||||
"Action-adventure Game (Media Genre)": Gaming,
|
||||
"programming": Technology,
|
||||
"sony interactive entertainment": Gaming,
|
||||
"dlc": Gaming,
|
||||
"diy": Learning,
|
||||
"bitcoin price": Blockchain,
|
||||
"monetary reform": Economics,
|
||||
"xbox360": Gaming,
|
||||
"zelda": Gaming,
|
||||
"Steam": Gaming,
|
||||
"capcom": Gaming,
|
||||
"economic": Economics,
|
||||
"simulation": Technology,
|
||||
"recession": Economics,
|
||||
"fortnite battle royale": Gaming,
|
||||
"altcoins": Blockchain,
|
||||
"facebook": Technology,
|
||||
"cryptocurrency news": Blockchain,
|
||||
"singleplayer": Gaming,
|
||||
"altcoin": Blockchain,
|
||||
"pacman": Gaming,
|
||||
"fashion": Art,
|
||||
"gta 5": Gaming,
|
||||
"Grand Theft Auto V": Gaming,
|
||||
"valve": Gaming,
|
||||
"Massively Multiplayer Online Role-playing Game (Video Game Genre)": Gaming,
|
||||
"#gaming": Gaming,
|
||||
"Bmw": Automotive,
|
||||
"halo": Gaming,
|
||||
"Hillary Clinton": News,
|
||||
"trading": Economics,
|
||||
"computer games": Gaming,
|
||||
"vehicle": Automotive,
|
||||
"desktop": Technology,
|
||||
"coin": Blockchain,
|
||||
"water": Nature,
|
||||
"nintendo switch": Gaming,
|
||||
"progressive talk": News,
|
||||
"3d games": Gaming,
|
||||
"animals": Nature,
|
||||
"install": Technology,
|
||||
"MOBA": Gaming,
|
||||
"Ps2": Gaming,
|
||||
"Role-playing Video Game (Media Genre)": Gaming,
|
||||
"trump": News,
|
||||
"ripple": Blockchain,
|
||||
"parody": Funny,
|
||||
"supercell": Gaming,
|
||||
"dota2": Gaming,
|
||||
"microsoft": Technology,
|
||||
"sims": Gaming,
|
||||
"barack obama": News,
|
||||
"funny video": Funny,
|
||||
"finance": Economics,
|
||||
"floral": Nature,
|
||||
"creative hairstyles": Art,
|
||||
"food holidays": Food,
|
||||
"colorful hair": Art,
|
||||
"currency": Economics,
|
||||
"free bitcoin": Blockchain,
|
||||
"how-to": Learning,
|
||||
"pc gaming": Gaming,
|
||||
"Playstation 4": Gaming,
|
||||
"twitter": Technology,
|
||||
"battle royale": Gaming,
|
||||
"rpg games": Gaming,
|
||||
"sims 4": Gaming,
|
||||
"Android": Technology,
|
||||
"Benz": Automotive,
|
||||
"silly": Funny,
|
||||
"roblox": Gaming,
|
||||
"Mw3": Gaming,
|
||||
"humour": Funny,
|
||||
"far cry 5": Gaming,
|
||||
"Mobile": Technology,
|
||||
"squad gameplay": Gaming,
|
||||
"tf2": Gaming,
|
||||
"Auto Show (Event)": Automotive,
|
||||
"business": Economics,
|
||||
"apple": Technology,
|
||||
"Ps3": Gaming,
|
||||
"Resident Evil": Gaming,
|
||||
"iOS": Technology,
|
||||
"Mercedes": Automotive,
|
||||
"engine": Automotive,
|
||||
"indie game": Gaming,
|
||||
"overwatch": Gaming,
|
||||
"brawl": Gaming,
|
||||
"ltc": Blockchain,
|
||||
"Call of Duty®: Black Ops III": Gaming,
|
||||
"Donald Trump": News,
|
||||
"dash": Blockchain,
|
||||
"newscast": News,
|
||||
"server": Technology,
|
||||
"gardening": Nature,
|
||||
"console": Gaming,
|
||||
"mugen": Gaming,
|
||||
"Play": Gaming,
|
||||
"Nvidia": Technology,
|
||||
"cool long hair": Art,
|
||||
"howto": Learning,
|
||||
"Mass effect": Gaming,
|
||||
"mercedes": Automotive,
|
||||
"ubisoft": Gaming,
|
||||
"open source": Technology,
|
||||
"Minecraft: PlayStation®4 Edition": Gaming,
|
||||
"rts": Gaming,
|
||||
"mmorpg": Gaming,
|
||||
"first person shooter": Gaming,
|
||||
"eos": Blockchain,
|
||||
"motorcycle": Automotive,
|
||||
"Action Role-playing Game (Video Game Genre)": Gaming,
|
||||
"steemit": Blockchain,
|
||||
"porsche": Automotive,
|
||||
"sport": Sports,
|
||||
"training": Sports,
|
||||
"Apple": Technology,
|
||||
"coinbase": Blockchain,
|
||||
"Role-playing Game (Game Genre)": Gaming,
|
||||
"sonic": Gaming,
|
||||
"advice": Learning,
|
||||
"videogames": Gaming,
|
||||
"Simulator": Gaming,
|
||||
"level": Gaming,
|
||||
"apps": Technology,
|
||||
"sims 3": Gaming,
|
||||
"Satire": Funny,
|
||||
"esports": Gaming,
|
||||
"Minecraft (Video Game)": Gaming,
|
||||
"simulator": Gaming,
|
||||
"horrible gamers": Gaming,
|
||||
"Bus": Automotive,
|
||||
"moba": Gaming,
|
||||
"football": Sports,
|
||||
"cryptocurrencies": Blockchain,
|
||||
"the game": Gaming,
|
||||
"laptop": Technology,
|
||||
"link": Gaming,
|
||||
"Game Reviews": Gaming,
|
||||
"sims 5": Gaming,
|
||||
"snk": Gaming,
|
||||
"online games": Gaming,
|
||||
"squad game": Gaming,
|
||||
"switch": Gaming,
|
||||
"Lets Play": Gaming,
|
||||
"racism": News,
|
||||
"jeep": Automotive,
|
||||
}
|
||||
+18
-2
@@ -1,6 +1,7 @@
|
||||
package thumbs
|
||||
|
||||
import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -21,6 +22,7 @@ type thumbnailUploader struct {
|
||||
}
|
||||
|
||||
const thumbnailPath = "/tmp/ytsync_thumbnails/"
|
||||
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
|
||||
|
||||
func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
_ = os.Mkdir(thumbnailPath, 0750)
|
||||
@@ -44,7 +46,7 @@ func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
}
|
||||
|
||||
func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
key := aws.String("/thumbnails/" + u.name)
|
||||
key := &u.name
|
||||
thumb, err := os.Open("/tmp/ytsync_thumbnails/" + u.name)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -62,8 +64,9 @@ func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
})
|
||||
u.mirroredUrl = "https://thumbnails.lbry.com/" + u.name
|
||||
u.mirroredUrl = ThumbnailEndpoint + u.name
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
@@ -92,3 +95,16 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
|
||||
return tu.mirroredUrl, nil
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
||||
if thumbnails.Maxres != nil {
|
||||
return thumbnails.Maxres
|
||||
} else if thumbnails.High != nil {
|
||||
return thumbnails.High
|
||||
} else if thumbnails.Medium != nil {
|
||||
return thumbnails.Medium
|
||||
} else if thumbnails.Standard != nil {
|
||||
return thumbnails.Standard
|
||||
}
|
||||
return thumbnails.Default
|
||||
}
|
||||
Reference in new issue
Block a user