Compare commits

...
16 Commits
Author SHA1 Message Date
Niko Storni 74d5e2606d fix cover URL
fix fallback download
fix thumbnail permissions
2019-05-28 22:40:31 +02:00
Niko Storni b8d2b9335b implement fallback downloader
fix bugs
update lbry.go
add tags manager/mapper
2019-05-28 19:28:35 +02:00
Niko Storni 637785f4ac Add prefilling for upgrades
add background image to channels
2019-05-24 19:01:16 +02:00
Niko Storni c9dffb0a82 rollback btcd libraries 2019-05-16 22:51:23 +02:00
Niko Storni 6ea0bd9834 update dependencies 2019-05-15 22:48:21 +02:00
Niko Storni 93986c53e0 lock to versioned lbry.go
fix github token
2019-05-10 20:12:45 +02:00
Niko Storni 843575c4ba add upgrade process for channels
add check for videos that don't need reprocessing
2019-05-09 17:07:01 +02:00
Niko Storni aecfc67f84 bug fixes
first working version of re-processing
add metadata version to api updates
2019-05-08 23:12:13 +02:00
Niko Storni 1f7017f3d0 videos never published should not be processed instead published 2019-05-07 22:07:44 +02:00
Niko Storni 75364805a5 more refactoring 2019-05-07 21:15:43 +02:00
Niko Storni 07f66a089f update dependency and fix bugs 2019-05-07 18:28:25 +02:00
Niko Storni 36abd1e471 remove lambda dependency
fix error checking
2019-05-07 10:01:11 -04:00
Niko Storni 502bfa924b change minimum refill amount
remove more berkeley code
add address reuse code
remove strPtr function in favor of utils
refactor code
2019-05-06 21:56:56 +02:00
Niko Storni 23b8fb4e54 add support for new SDK
add support for direct thumbnails upload
remove UCB code
add support for republishing
refactoring
2019-05-03 05:11:52 +02:00
Niko Storni a648287e1c add unpublished status 2019-04-25 22:08:07 +02:00
Niko Storni af13ec5684 fixes for 0.36 2019-04-25 07:16:45 -04:00
11 changed files with 1258 additions and 449 deletions

No files matched your search

+1 -1
View File
@@ -9,7 +9,7 @@ env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "+G0TxyGyRYJb0L0qCCn6NPregigwhalkS9+/3mkV41xoAwUlEU9aclSJQVKKqeMrV0PkmJ6KrzYD78tnRj8Q5ZDe93LBDbc/WhXZ91JbRmIXIsVktz/78R2a2Q0DBrXMlyWXS84QouRfI3eyxf2BusERSYdr3H6BL4r15PIHJEmLGOchHsHwSI46PRGag+IFnbUb+fyUSs7er6gKeSp4+ZK1bSUgxiMUQrZfImYI9LS+cgqpDh67pX5jRbDNBihZZUwfreToYw3RbIAom4i2oix1rBBD/jpcTwO+WdyXFtur8RYGnsknePU88KiAB9bcXU4T2xVxR9wusK6jX5G2BChEOG9PjuVfeDqXBnskOEtAevwAKJRYW6ATmXqQZ+TAy1DFxsUWPVS7ykIA+69n7HzE7xu50AB1U2n6myT977i4jhBzw3VrB9NQz//4b2j6MCG1PhIgB38kffQWVsa/CF4RiiWi6a7ePRsPP0HdRUbXFQJWPevKgVJ9ve/ArQKiea1rP9zy9KdFAP30vESBkn4WMOCjfg7F6JbfV3e1ywBq9XMznj14iAZHwXR1tpPp0GekuMlVJHJWa0Ea/dfDGWshnr1Be71vc9T118w7sd0v1Dp4EQCFa4B0Hx7Ual7uWgeuweL/JLr1Am3BgWVcXOWrhzMT0b1tGFa9ZNz2Wwk="
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
deploy:
provider: script
skip_cleanup: true
+1 -5
View File
@@ -5,8 +5,6 @@ require (
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/aws/aws-sdk-go v1.17.3
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b // indirect
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 // indirect
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/go-ini/ini v1.42.0 // indirect
github.com/golang/protobuf v1.3.1 // indirect
@@ -17,8 +15,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 // indirect
github.com/lbryio/lbry.go v1.0.6
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
@@ -37,7 +34,6 @@ require (
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be // indirect
google.golang.org/api v0.3.2
google.golang.org/appengine v1.5.0 // indirect
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
+19 -18
View File
@@ -9,6 +9,7 @@ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -21,19 +22,17 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b h1:7J7sEce3LgtbMgs7PKcN61gF3b4txM6SjaRoJTSk640=
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 h1:GbQHMJ2u/geMPV1tbN7i7zARSoPAPuXWa44V0KYvJXU=
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
@@ -101,9 +100,11 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
@@ -112,6 +113,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -125,13 +127,13 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18 h1:ZhWjtvaq5r7julhcF9OSgx4bLv9UsdIx27zH1/fbrEc=
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18/go.mod h1:kd08aOMCuBVYJ3EafY4Kx3dRAWWQYhobJ9beREgsaRI=
github.com/lbryio/lbry.go v1.0.6 h1:dGZHwYdD5bb7PCOhG2ksJuvhOjqIARAcXJGggaOhC6o=
github.com/lbryio/lbry.go v1.0.6/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a h1:twWvrsBDvSb+qnmpSq3nvFrodgC5PpXUipyo4T/W790=
github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 h1:iERWR8Dkng30eRInI7gzolTEJBW9nBSK/sT+Z9aSipI=
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
@@ -178,11 +180,11 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
@@ -211,7 +213,6 @@ go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
@@ -249,8 +250,8 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be h1:mI+jhqkn68ybP0ORJqunXn+fq+Eeb4hHKqLQcFICjAc=
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -279,7 +280,6 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -287,7 +287,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -298,6 +297,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+21 -12
View File
@@ -2,6 +2,8 @@ package manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"strings"
"syscall"
"time"
@@ -70,21 +72,23 @@ func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool
}
const (
StatusPending = "pending" // waiting for permission to sync
StatusPendingEmail = "pendingemail" // permission granted but missing email
StatusQueued = "queued" // in sync queue. will be synced soon
StatusSyncing = "syncing" // syncing now
StatusSynced = "synced" // done
StatusFailed = "failed"
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
StatusPending = "pending" // waiting for permission to sync
StatusPendingEmail = "pendingemail" // permission granted but missing email
StatusQueued = "queued" // in sync queue. will be synced soon
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
StatusSyncing = "syncing" // syncing now
StatusSynced = "synced" // done
StatusFailed = "failed"
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
)
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUnpublished = "unpublished"
)
func (s *SyncManager) Start() error {
@@ -206,7 +210,12 @@ func (s *SyncManager) Start() error {
}
return nil
}
func (s *SyncManager) GetS3AWSConfig() aws.Config {
return aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
Region: &s.awsS3Region,
}
}
func (s *SyncManager) checkUsedSpace() error {
usedPctile, err := GetUsedSpace(s.blobsDir)
if err != nil {
+90 -43
View File
@@ -7,8 +7,6 @@ import (
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
@@ -21,6 +19,27 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 3
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accounts := accountsResponse.LBCMainnet
if os.Getenv("REGTEST") == "true" {
accounts = accountsResponse.LBCRegtest
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: 1000,
})
if err != nil {
return errors.Err(err)
}
}
return nil
}
func (s *Sync) walletSetup() error {
//prevent unnecessary concurrent execution
s.walletMux.Lock()
@@ -42,16 +61,11 @@ func (s *Sync) walletSetup() error {
}
log.Debugf("Starting balance is %.4f", balance)
var numOnSource int
if s.LbryChannelName == "@UCBerkeley" {
numOnSource = 10104
} else {
n, err := s.CountVideos()
if err != nil {
return err
}
numOnSource = int(n)
n, err := s.CountVideos()
if err != nil {
return err
}
numOnSource := int(n)
log.Debugf("Source channel has %d videos", numOnSource)
if numOnSource == 0 {
@@ -68,9 +82,18 @@ func (s *Sync) walletSetup() error {
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if numPublished > numOnSource && balance < 1 {
if s.Manager.syncStatus == StatusPendingUpgrade {
videosToUpgrade := 0
for _, v := range s.syncedVideos {
if v.Published && v.MetadataVersion < 2 {
videosToUpgrade++
}
}
minBalance += float64(videosToUpgrade) * 0.001
}
if numPublished > numOnSource && balance < minimumRefillAmount {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = 1 //since we ended up in this function it means some juice is still needed
minBalance = minimumRefillAmount
}
amountToAdd := minBalance - balance
@@ -83,8 +106,8 @@ func (s *Sync) walletSetup() error {
}
if amountToAdd > 0 {
if amountToAdd < 1 {
amountToAdd = 1 // no reason to bother adding less than 1 credit
if amountToAdd < minimumRefillAmount {
amountToAdd = minimumRefillAmount
}
err := s.addCredits(amountToAdd)
if err != nil {
@@ -98,7 +121,7 @@ func (s *Sync) walletSetup() error {
} else if claimAddress == nil {
return errors.Err("could not get unused address")
}
s.claimAddress = string((*claimAddress)[0])
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
if s.claimAddress == "" {
return errors.Err("found blank claim address")
}
@@ -122,7 +145,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
defaultAccount := ""
for _, account := range accountsNet {
if account.IsDefaultAccount {
if account.IsDefault {
defaultAccount = account.ID
break
}
@@ -246,9 +269,11 @@ func (s *Sync) ensureChannelOwnership() error {
}
}
}
channelUsesOldMetadata := false
if len((*channels).Items) == 1 {
channel := ((*channels).Items)[0]
if channel.Name == s.LbryChannelName {
channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
//TODO: eventually get rid of this when the whole db is filled
if s.lbryChannelID == "" {
err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
@@ -256,7 +281,9 @@ func (s *Sync) ensureChannelOwnership() error {
return errors.Err("the channel in the wallet is different than the channel in the database")
}
s.lbryChannelID = channel.ClaimID
return err
if !channelUsesOldMetadata {
return err
}
} else {
return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
}
@@ -290,7 +317,7 @@ func (s *Sync) ensureChannelOwnership() error {
return errors.Prefix("error creating YouTube service", err)
}
response, err := service.Channels.List("snippet").Id(s.YoutubeChannelID).Do()
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
if err != nil {
return errors.Prefix("error getting channel details", err)
}
@@ -300,24 +327,23 @@ func (s *Sync) ensureChannelOwnership() error {
}
channelInfo := response.Items[0].Snippet
channelBranding := response.Items[0].BrandingSettings
thumbnail := channelInfo.Thumbnails.Default
if channelInfo.Thumbnails.Maxres != nil {
thumbnail = channelInfo.Thumbnails.Maxres
} else if channelInfo.Thumbnails.High != nil {
thumbnail = channelInfo.Thumbnails.High
} else if channelInfo.Thumbnails.Medium != nil {
thumbnail = channelInfo.Thumbnails.Medium
} else if channelInfo.Thumbnails.Standard != nil {
thumbnail = channelInfo.Thumbnails.Standard
}
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, aws.Config{
Credentials: credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, ""),
Region: &s.AwsS3Region,
})
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
if err != nil {
return err
}
var bannerURL *string
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
if err != nil {
return err
}
bannerURL = &bURL
}
var languages []string = nil
if channelInfo.DefaultLanguage != "" {
languages = []string{channelInfo.DefaultLanguage}
@@ -326,16 +352,37 @@ func (s *Sync) ensureChannelOwnership() error {
if channelInfo.Country != "" {
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
}
c, err := s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
})
var c *jsonrpc.TransactionSummary
if channelUsesOldMetadata {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
CoverURL: bannerURL,
},
})
} else {
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
CoverURL: bannerURL,
})
}
if err != nil {
return err
}
+51 -67
View File
@@ -1,11 +1,7 @@
package manager
import (
"bufio"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
@@ -21,6 +17,7 @@ import (
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/sources"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
@@ -51,7 +48,7 @@ type video interface {
IDAndNum() string
PlaylistPosition() int
PublishedAt() time.Time
Sync(*jsonrpc.Client, string, float64, string, int, *namer.Namer, float64) (*sources.SyncSummary, error)
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool) (*sources.SyncSummary, error)
}
// sorting videos
@@ -319,12 +316,14 @@ func (s *Sync) FullCycle() (e error) {
return nil
}
func (s *Sync) setChannelTerminationStatus(e *error) {
if *e != nil {
//conditions for which a channel shouldn't be marked as failed
noFailConditions := []string{
"this youtube channel is being managed by another server",
"interrupted during daemon startup",
"playlist items not found",
}
if util.SubstringInSlice((*e).Error(), noFailConditions) {
return
@@ -390,20 +389,24 @@ func logShutdownError(shutdownErr error) {
var thumbnailHosts = []string{
"berk.ninja/thumbnails/",
"https://thumbnails.lbry.com/",
thumbs.ThumbnailEndpoint,
}
func isYtsyncClaim(c jsonrpc.Claim) bool {
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
return false
}
if c.Value.GetStream().GetThumbnailUrl() == "" {
if c.Value.GetThumbnail() == nil || c.Value.GetThumbnail().GetUrl() == "" {
//most likely a claim created outside of ytsync, ignore!
return false
}
return util.InSlice(c.Value.GetStream().GetThumbnailUrl(), thumbnailHosts)
for _, th := range thumbnailHosts {
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
return true
}
}
return false
}
// fixDupes abandons duplicate claims
@@ -414,7 +417,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
if !isYtsyncClaim(c) {
continue
}
tn := c.Value.GetStream().GetThumbnailUrl()
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID)
@@ -450,7 +453,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
}
count++
//check if claimID is in remote db
tn := c.Value.GetStream().GetThumbnailUrl()
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
pv, ok := s.syncedVideos[videoID]
if !ok || pv.ClaimName != c.Name {
@@ -481,7 +484,10 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
}
func (s *Sync) doSync() error {
var err error
err := s.enableAddressReuse()
if err != nil {
return errors.Prefix("could not set address reuse policy", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
@@ -545,7 +551,7 @@ func (s *Sync) doSync() error {
}
if s.LbryChannelName == "@UCBerkeley" {
err = s.enqueueUCBVideos()
err = errors.Err("UCB is not supported on this version of YTSYNC")
} else {
err = s.enqueueYoutubeVideos()
}
@@ -613,6 +619,7 @@ func (s *Sync) startWorker(workerNum int) {
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -643,8 +650,20 @@ func (s *Sync) startWorker(workerNum int) {
}
SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
}
existingClaim, ok := s.syncedVideos[v.ID()]
existingClaimID := ""
existingClaimName := ""
existingClaimSize := v.Size()
if ok {
existingClaimID = existingClaim.ClaimID
existingClaimName = existingClaim.ClaimName
if existingClaim.Size > 0 {
existingClaimSize = &existingClaim.Size
}
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, "", "", err.Error(), v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@@ -713,14 +732,14 @@ func (s *Sync) enqueueYoutubeVideos() error {
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
videoIDs[i] = item.Snippet.ResourceId.VideoId
}
req2 := service.Videos.List("snippet,contentDetails").Id(strings.Join(videoIDs[:], ","))
req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
videosListResponse, err := req2.Do()
if err != nil {
return errors.Prefix("error getting videos info", err)
}
for _, item := range videosListResponse.Items {
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position))
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig()))
}
log.Infof("Got info for %d videos from youtube API", len(videos))
@@ -752,55 +771,6 @@ Enqueue:
return nil
}
func (s *Sync) enqueueUCBVideos() error {
var videos []video
csvFile, err := os.Open("ucb.csv")
if err != nil {
return err
}
reader := csv.NewReader(bufio.NewReader(csvFile))
for {
line, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
return err
}
data := struct {
PublishedAt string `json:"publishedAt"`
}{}
err = json.Unmarshal([]byte(line[4]), &data)
if err != nil {
return err
}
videos = append(videos, sources.NewUCBVideo(line[0], line[2], line[1], line[3], data.PublishedAt, s.videoDirectory))
}
log.Printf("Publishing %d videos\n", len(videos))
sort.Sort(byPublishedAt(videos))
Enqueue:
for _, v := range videos {
select {
case <-s.grp.Ch():
break Enqueue
default:
}
select {
case s.queue <- v:
case <-s.grp.Ch():
break Enqueue
}
}
return nil
}
func (s *Sync) processVideo(v video) (err error) {
defer func() {
if p := recover(); p != nil {
@@ -823,6 +793,7 @@ func (s *Sync) processVideo(v video) (err error) {
sv, ok := s.syncedVideos[v.ID()]
s.syncedVideosMux.RUnlock()
alreadyPublished := ok && sv.Published
isUpgradeSync := s.Manager.syncStatus == StatusPendingUpgrade
neverRetryFailures := []string{
"Error extracting sts from embedded url response",
@@ -838,10 +809,15 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
if alreadyPublished {
if alreadyPublished && !isUpgradeSync {
log.Println(v.ID() + " already published")
return nil
}
newMetadataVersion := int8(2)
if ok && sv.MetadataVersion >= newMetadataVersion {
log.Println(v.ID() + " upgraded to the new metadata")
return nil
}
if v.PlaylistPosition() > s.Manager.videosLimit {
log.Println(v.ID() + " is old: skipping")
@@ -851,8 +827,16 @@ func (s *Sync) processVideo(v video) (err error) {
if err != nil {
return err
}
sp := sources.SyncParams{
ClaimAddress: s.claimAddress,
Amount: publishAmount,
ChannelID: s.lbryChannelID,
MaxVideoSize: s.Manager.maxVideoSize,
Namer: s.namer,
MaxVideoLength: s.Manager.maxVideoLength,
}
summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.lbryChannelID, s.Manager.maxVideoSize, s.namer, s.Manager.maxVideoLength)
summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync)
if err != nil {
return err
}
+11 -5
View File
@@ -76,10 +76,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
}
type SyncedVideo struct {
VideoID string `json:"video_id"`
Published bool `json:"published"`
FailureReason string `json:"failure_reason"`
ClaimName string `json:"claim_name"`
VideoID string `json:"video_id"`
Published bool `json:"published"`
FailureReason string `json:"failure_reason"`
ClaimName string `json:"claim_name"`
ClaimID string `json:"claim_id"`
Size int64 `json:"size"`
MetadataVersion int8 `json:"metadata_version"`
}
func sanitizeFailureReason(s *string) {
@@ -121,7 +124,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
claimNames := make(map[string]bool)
for _, v := range response.Data {
svs[v.VideoID] = v
claimNames[v.ClaimName] = v.Published
if v.ClaimName != "" {
claimNames[v.ClaimName] = v.Published
}
}
return svs, claimNames, nil
}
@@ -178,6 +183,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", "2")
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
}
-219
View File
@@ -1,219 +0,0 @@
package sources
import (
"net/http"
"os"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/ytsync/namer"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
log "github.com/sirupsen/logrus"
)
type ucbVideo struct {
id string
title string
channel string
description string
publishedAt time.Time
dir string
claimNames map[string]bool
syncedVideosMux *sync.RWMutex
}
func NewUCBVideo(id, title, channel, description, publishedAt, dir string) *ucbVideo {
p, _ := time.Parse(time.RFC3339Nano, publishedAt) // ignore parse errors
return &ucbVideo{
id: id,
title: title,
description: description,
channel: channel,
dir: dir,
publishedAt: p,
}
}
func (v *ucbVideo) ID() string {
return v.id
}
func (v *ucbVideo) PlaylistPosition() int {
return 0
}
func (v *ucbVideo) IDAndNum() string {
return v.ID() + " (?)"
}
func (v *ucbVideo) PublishedAt() time.Time {
return v.publishedAt
//r := regexp.MustCompile(`(\d\d\d\d)-(\d\d)-(\d\d)`)
//matches := r.FindStringSubmatch(v.title)
//if len(matches) > 0 {
// year, _ := strconv.Atoi(matches[1])
// month, _ := strconv.Atoi(matches[2])
// day, _ := strconv.Atoi(matches[3])
// return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
//}
//return time.Now()
}
func (v *ucbVideo) getFilename() string {
return v.dir + "/" + v.id + ".mp4"
}
func (v *ucbVideo) getClaimName(attempt int) string {
reg := regexp.MustCompile(`[^a-zA-Z0-9]+`)
suffix := ""
if attempt > 1 {
suffix = "-" + strconv.Itoa(attempt)
}
maxLen := 40 - len(suffix)
chunks := strings.Split(strings.ToLower(strings.Trim(reg.ReplaceAllString(v.title, "-"), "-")), "-")
name := chunks[0]
if len(name) > maxLen {
return name[:maxLen]
}
for _, chunk := range chunks[1:] {
tmpName := name + "-" + chunk
if len(tmpName) > maxLen {
if len(name) < 20 {
name = tmpName[:maxLen]
}
break
}
name = tmpName
}
return name + suffix
}
func (v *ucbVideo) getAbbrevDescription() string {
maxLines := 10
description := strings.TrimSpace(v.description)
if strings.Count(description, "\n") < maxLines {
return description
}
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..."
}
func (v *ucbVideo) download() error {
videoPath := v.getFilename()
_, err := os.Stat(videoPath)
if err != nil && !os.IsNotExist(err) {
return err
} else if err == nil {
log.Debugln(v.id + " already exists at " + videoPath)
return nil
}
creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "")
s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds})
if err != nil {
return err
}
downloader := s3manager.NewDownloader(s)
out, err := os.Create(videoPath)
if err != nil {
return err
}
defer out.Close()
log.Println("lbry-niko2/videos/" + v.channel + "/" + v.id)
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String("lbry-niko2"),
Key: aws.String("/videos/" + v.channel + "/" + v.id + ".mp4"),
})
if err != nil {
return err
} else if bytesWritten == 0 {
return errors.Err("zero bytes written")
}
return nil
}
func (v *ucbVideo) saveThumbnail() error {
resp, err := http.Get("https://s3.us-east-2.amazonaws.com/lbry-niko2/thumbnails/" + v.id)
if err != nil {
return err
}
defer resp.Body.Close()
creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "")
s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds})
if err != nil {
return err
}
uploader := s3manager.NewUploader(s)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("berk.ninja"),
Key: aws.String("thumbnails/" + v.id),
ContentType: aws.String("image/jpeg"),
Body: resp.Body,
})
return err
}
func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
options := jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: v.title,
Description: v.getAbbrevDescription(),
ClaimAddress: &claimAddress,
Languages: []string{"en"},
ThumbnailURL: strPtr("https://berk.ninja/thumbnails/" + v.id),
Tags: []string{},
},
Author: strPtr("UC Berkeley"),
License: strPtr("see description"),
StreamType: &jsonrpc.StreamTypeVideo,
ChannelID: &channelID,
}
return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer)
}
func (v *ucbVideo) Size() *int64 {
return nil
}
func (v *ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) {
//download and thumbnail can be done in parallel
err := v.download()
if err != nil {
return nil, errors.Prefix("download error", err)
}
log.Debugln("Downloaded " + v.id)
//err = v.SaveThumbnail()
//if err != nil {
// return errors.WrapPrefix(err, "thumbnail error", 0)
//}
//log.Debugln("Created thumbnail for " + v.id)
summary, err := v.publish(daemon, claimAddress, amount, channelID, namer)
if err != nil {
return nil, errors.Prefix("publish error", err)
}
return summary, nil
}
+228 -77
View File
@@ -1,12 +1,12 @@
package sources
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"math"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
@@ -15,9 +15,14 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
"github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
@@ -36,9 +41,49 @@ type YoutubeVideo struct {
dir string
youtubeInfo *youtube.Video
tags []string
awsConfig aws.Config
thumbnailURL string
lbryChannelID string
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64) *YoutubeVideo {
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "Film & Animation",
"2": "Autos & Vehicles",
"10": "Music",
"15": "Pets & Animals",
"17": "Sports",
"18": "Short Movies",
"19": "Travel & Events",
"20": "Gaming",
"21": "Videoblogging",
"22": "People & Blogs",
"23": "Comedy",
"24": "Entertainment",
"25": "News & Politics",
"26": "Howto & Style",
"27": "Education",
"28": "Science & Technology",
"29": "Nonprofits & Activism",
"30": "Movies",
"31": "Anime/Animation",
"32": "Action/Adventure",
"33": "Classics",
"34": "Comedy",
"35": "Documentary",
"36": "Drama",
"37": "Family",
"38": "Foreign",
"39": "Horror",
"40": "Sci-Fi/Fantasy",
"41": "Thriller",
"42": "Shorts",
"43": "Shows",
"44": "Trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
return &YoutubeVideo{
id: videoData.Id,
@@ -49,6 +94,7 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
publishedAt: publishedAt,
dir: directory,
youtubeInfo: videoData,
awsConfig: awsConfig,
}
}
@@ -101,9 +147,30 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
if strings.Count(description, "\n") < maxLines {
return description
}
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
if v.lbryChannelID == khanAcademyClaimID {
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
}
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..."
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+bestaudio/best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"))
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
return errors.Err(err)
}
return nil
}
func (v *YoutubeVideo) download() error {
videoPath := v.getFullPath()
@@ -190,108 +257,108 @@ func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) videoDir() string {
return v.dir + "/" + v.id
}
func (v *YoutubeVideo) getDownloadedPath() (string, error) {
files, err := ioutil.ReadDir(v.videoDir())
log.Infoln(v.videoDir())
if err != nil {
err = errors.Prefix("list error", err)
log.Errorln(err)
return "", err
}
for _, f := range files {
if f.IsDir() {
continue
}
if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
return v.videoDir() + "/" + f.Name(), nil
}
}
return "", errors.Err("could not find any downloaded videos")
}
func (v *YoutubeVideo) delete() error {
videoPath := v.getFullPath()
err := os.Remove(videoPath)
videoPath, err := v.getDownloadedPath()
if err != nil {
log.Errorln(errors.Prefix("delete error", err))
log.Errorln(err)
return err
}
log.Debugln(v.id + " deleted from disk (" + videoPath + ")")
return nil
}
err = os.Remove(videoPath)
log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
func (v *YoutubeVideo) triggerThumbnailSave() error {
client := &http.Client{Timeout: 30 * time.Second}
params, err := json.Marshal(map[string]string{"videoid": v.id})
if err != nil {
err = errors.Prefix("delete error", err)
log.Errorln(err)
return err
}
request, err := http.NewRequest(http.MethodPut, "https://jgp4g1qoud.execute-api.us-east-1.amazonaws.com/prod/thumbnail", bytes.NewBuffer(params))
if err != nil {
return err
}
response, err := client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
contents, err := ioutil.ReadAll(response.Body)
if err != nil {
return err
}
var decoded struct {
Error int `json:"error"`
Url string `json:"url,omitempty"`
Message string `json:"message,omitempty"`
}
err = json.Unmarshal(contents, &decoded)
if err != nil {
return err
}
if decoded.Error != 0 {
return errors.Err("error creating thumbnail: " + decoded.Message)
}
return nil
}
func strPtr(s string) *string { return &s }
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
return err
}
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
if channelID == khanAcademyClaimID {
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
}
var languages []string = nil
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, namer *namer.Namer) (*SyncSummary, error) {
languages, locations, tags := v.getMetadata()
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
options := jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: v.title,
Description: v.getAbbrevDescription() + additionalDescription,
Description: v.getAbbrevDescription(),
ClaimAddress: &claimAddress,
Languages: languages,
ThumbnailURL: strPtr("https://thumbnails.lbry.com/" + v.id),
Tags: v.youtubeInfo.Snippet.Tags,
ThumbnailURL: &v.thumbnailURL,
Tags: tags,
Locations: locations,
},
Author: strPtr(v.channelTitle),
License: strPtr("Copyrighted (contact author)"),
StreamType: &jsonrpc.StreamTypeVideo,
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &channelID,
Author: util.PtrToString(v.channelTitle),
License: util.PtrToString("Copyrighted (contact author)"),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
ChannelID: &v.lbryChannelID,
}
return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
downloadPath, err := v.getDownloadedPath()
if err != nil {
return nil, err
}
return publishAndRetryExistingNames(daemon, v.title, downloadPath, amount, options, namer)
}
func (v *YoutubeVideo) Size() *int64 {
return v.size
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) {
v.maxVideoSize = int64(maxVideoSize) * 1024 * 1024
v.maxVideoLength = maxVideoLength
//download and thumbnail can be done in parallel
type SyncParams struct {
ClaimAddress string
Amount float64
ChannelID string
MaxVideoSize int
Namer *namer.Namer
MaxVideoLength float64
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
if reprocess && existingVideoData != nil && existingVideoData.Published {
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, err
}
return v.downloadAndPublish(daemon, params)
}
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
if err != nil {
return nil, errors.Prefix("download error", err)
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
fallBackErr := v.fallbackDownload()
if fallBackErr != nil {
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
return nil, errors.Prefix("download error", err) //return original error
}
}
log.Debugln("Downloaded " + v.id)
@@ -301,9 +368,93 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount
}
log.Debugln("Created thumbnail for " + v.id)
summary, err := v.publish(daemon, claimAddress, amount, channelID, namer)
summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.Namer)
//delete the video in all cases (and ignore the error)
_ = v.delete()
return summary, errors.Prefix("publish error", err)
}
func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Location, tags []string) {
languages = nil
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}
locations = nil
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
locations = []jsonrpc.Location{{
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
}}
}
tags = append([]string{youtubeCategories[v.youtubeInfo.Snippet.CategoryId]}, v.youtubeInfo.Snippet.Tags...)
tags = tagsManager.SanitizeTags(tags)
return languages, locations, tags
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
if err != nil {
return nil, errors.Err(err)
}
if len(c.Claims) == 0 {
return nil, errors.Err("cannot reprocess: no claim found for this video")
} else if len(c.Claims) > 1 {
return nil, errors.Err("cannot reprocess: too many claims. claimID: %s", existingVideoData.ClaimID)
}
currentClaim := c.Claims[0]
languages, locations, tags := v.getMetadata()
thumbnailURL := ""
if currentClaim.Value.GetThumbnail() == nil {
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
} else {
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
}
videoSize, err := currentClaim.GetStreamSizeByMagic()
if err != nil {
if existingVideoData.Size > 0 {
videoSize = uint64(existingVideoData.Size)
} else {
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
return nil, errors.Err("the video must be republished as we can't get the right size")
}
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: v.title,
Description: v.getAbbrevDescription(),
Tags: tags,
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact author)"),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &v.lbryChannelID,
},
FileSize: &videoSize,
})
if err != nil {
return nil, err
}
return &SyncSummary{
ClaimID: pr.Outputs[0].ClaimID,
ClaimName: pr.Outputs[0].Name,
}, nil
}
+818
View File
@@ -0,0 +1,818 @@
package tagsManager
const (
Gaming = "gaming"
Blockchain = "blockchain"
News = "news"
Learning = "learning"
Funny = "funny"
Technology = "technology"
Automotive = "automotive"
Economics = "economics"
Sports = "sports"
Food = "food"
Science = "science"
Fitness = "fitness"
Art = "art"
Nature = "nature"
)
func SanitizeTags(tags []string) []string {
ts := &tagsSanitizer{
Unsanitized: tags,
}
ts.init()
ts.cleanup()
ts.replace()
ts.add()
sanitizedTags := make([]string, len(ts.Sanitized))
index := 0
for t := range ts.Sanitized {
sanitizedTags[index] = t
index++
}
return sanitizedTags
}
type tagsSanitizer struct {
Unsanitized []string
Sanitized map[string]bool
}
func (ts *tagsSanitizer) init() {
if len(ts.Sanitized) == 0 {
ts.Sanitized = make(map[string]bool, len(ts.Unsanitized)+14)
}
}
func (ts *tagsSanitizer) cleanup() {
for _, t := range ts.Unsanitized {
_, ok := tagsToSkip[t]
if !ok {
ts.Sanitized[t] = true
}
}
}
func (ts *tagsSanitizer) replace() {
for _, t := range ts.Unsanitized {
match, filterMatch := mapAndReplace[t]
if filterMatch {
delete(ts.Sanitized, t)
ts.Sanitized[match] = true
}
}
}
func (ts *tagsSanitizer) add() {
for _, t := range ts.Unsanitized {
match, filterMatch := mapAndKeep[t]
if filterMatch {
ts.Sanitized[match] = true
ts.Sanitized[t] = true
}
}
}
var tagsToSkip = map[string]*struct{}{
"video": nil,
"review": nil,
"youtube": nil,
"david packman": nil,
"2018": nil,
"new": nil,
"the": nil,
"music": nil,
"hd": nil,
"talk": nil,
"entertainment": nil,
"Radio": nil,
"2017": nil,
"live": nil,
"free": nil,
"america": nil,
"of": nil,
"david pakman": nil,
"dave pacman": nil,
"pakman": nil,
"davidpakman.com": nil,
"tdps": nil,
"1080p": nil,
"united": nil,
"david pakman show": nil,
"states": nil,
"europe": nil,
"tomonews": nil,
"Tomonews": nil,
"2016": nil,
"eeuu": nil,
"REVIEW": nil,
"Launch": nil,
"taiwanese animators": nil,
"best": nil,
"TURBO": nil,
"television": nil,
"gt": nil,
"USA": nil,
"GPS": nil,
"PREMIERE": nil,
"guide": nil,
"scoope": nil,
"US": nil,
"nma": nil,
"Nma": nil,
"to": nil,
"auto imagen": nil,
"tips": nil,
"action": nil,
"download": nil,
"david pacman": nil,
"taiwanese animation": nil,
"Taiwanese animation": nil,
"YouTube Editor": nil,
"Tomo news": nil,
"replay": nil,
"update": nil,
"battle": nil,
"2019": nil,
"story": nil,
"one": nil,
"spreaker": nil,
"Jay's": nil,
"episode": nil,
"for": nil,
"SHAREfactory™": nil,
"#hangoutsonair": nil,
"#hoa": nil,
"vs": nil,
"bob": nil,
"dnb portal": nil,
"instagram": nil,
"challenge": nil,
"eatmydiction1": nil,
"usa": nil,
"français": nil,
"live stream": nil,
"top": nil,
"fr": nil,
"fanta": nil,
"HD": nil,
"and": nil,
"foto": nil,
"no": nil,
"720p": nil,
"up": nil,
"radio": nil,
"david": nil,
"russia": nil,
"ytquality=high": nil,
"part": nil,
"bollywood tashan": nil,
"bounty": nil,
"dnb": nil,
"highlights": nil,
"thefantasio974": nil,
"great": nil,
"man": nil,
"replica": nil,
"red": nil,
"chiara ferragni": nil,
"in": nil,
"bob lennon": nil,
"series": nil,
"life": nil,
"first": nil,
"dnbportal": nil,
"fantabobgames": nil,
"segui": nil,
"lennon": nil,
"neurofunk": nil,
"on": nil,
"kag3": nil,
"2015": nil,
"ITA": nil,
"gamerworf": nil,
"chickens": nil,
"uct-wqktykk1_70u4bb4k4lq": nil,
"beard": nil,
"videos": nil,
"nidge": nil,
"bill still": nil,
"nickatnyte": nil,
"прохождение": nil,
"minimal": nil,
"clash": nil,
"trofeo": nil,
"attack": nil,
"test": nil,
"DFX": nil,
"india": nil,
"police": nil,
"campaign": nil,
"tricks": nil,
"depression": nil,
"unique": nil,
"Bharat": nil,
"imagen": nil,
"LanceScurv": nil,
"us": nil,
"google": nil,
"defense": nil,
"gameplay fr": nil,
"giveaway": nil,
"hot": nil,
"Interior": nil,
"show": nil,
"YouTube Capture": nil,
"jump": nil,
"2014": nil,
"thailand": nil,
"santa": nil,
"ultimate": nil,
"deutsch": nil,
"mission": nil,
"The Lancescurv Show": nil,
"Legends": nil,
"modi": nil,
"noise": nil,
"Texas": nil,
"china": nil,
"Scurv": nil,
"gift": nil,
"sikh": nil,
"night": nil,
"Episode": nil,
"good": nil,
"plus": nil,
"szekelyvegan": nil,
"székelyvegán": nil,
"szekely": nil,
"игра": nil,
"is": nil,
"székely vegán": nil,
"john sonmez": nil,
"jew": nil,
"magyar vlogger": nil,
"magyar vlog": nil,
"hungarian vlog": nil,
"baba": nil,
"montage": nil,
"New": nil,
"squad": nil,
"german": nil,
"amerika": nil,
"full": nil,
"sri": nil,
"kristomaster4": nil,
"narendra": nil,
"sant": nil,
"adityanath": nil,
"sadhguru": nil,
"ped": nil,
"Sadhu": nil,
"Shyam": nil,
"dharm": nil,
"babas": nil,
"gaay": nil,
"Pradesh": nil,
"Raebareli": nil,
"waale": nil,
"Uttar": nil,
"Bandar": nil,
"kutta": nil,
"bakri": nil,
"aughad": nil,
"suar": nil,
"jaanwar": nil,
"tona": nil,
"bhawreshwara": nil,
"vaanar": nil,
"bhagwa": nil,
"prakriti": nil,
"samaj": nil,
"totka": nil,
"bhoj": nil,
"dave": nil,
"vulcanhdgaming": nil,
"fast": nil,
"moon": nil,
"scene": nil,
"tv": nil,
"2": nil,
"fighter": nil,
"NEW": nil,
"jeux": nil,
"игры": nil,
"tanyázás": nil,
"obiettivo": nil,
"csgo": nil,
"The guy From Pittsburgh": nil,
"blue": nil,
"youtuber": nil,
"clash royale": nil,
"SHOW": nil,
"johnsp69": nil,
"japan": nil,
"YouTube": nil,
"The David Pakman Show": nil,
"thecreativeone": nil,
"Youtube": nil,
"classic": nil,
"Talk Radio": nil,
"white": nil,
"летсплей": nil,
"clash of clans": nil,
"Industry (Organization Sector)": nil,
"difranco": nil,
"custom": nil,
"прохождение игры": nil,
"progressive podcast": nil,
"george Senda": nil,
"360": nil,
"betv": nil,
"gamingoncaffeine": nil,
"Drive": nil,
"Africa": nil,
"progressive news": nil,
"full hd": nil,
"Asia": nil,
"liberal news": nil,
"packman": nil,
"hdtv": nil,
"Entertainment": nil,
"Simple Programmer": nil,
"indonesia": nil,
"karmakut": nil,
"gorthemoviegod": nil,
"Lance scurvin": nil,
"silver": nil,
"clans": nil,
"игры для мальчиков": nil,
"игры для девочек": nil,
"blackpeace72": nil,
"multicolored": nil,
"intro": nil,
"creative beard": nil,
"cyber locks": nil,
"uniqornaments": nil,
"daily holidays": nil,
"daily celebration": nil,
"holiday everyday": nil,
"3d": nil,
"lp": nil,
"interview": nil,
"Heroes": nil,
"maskedmage": nil,
"рпг": nil,
"north": nil,
"road": nil,
"monster": nil,
"de": nil,
"mode": nil,
"podcastradio": nil,
"Man": nil,
"preview": nil,
"rage": nil,
"map": nil,
"Pittsburgh pa": nil,
"david di franco": nil,
"win": nil,
"junior senior": nil,
"demo": nil,
"paul": nil,
"simpleprogrammer.com": nil,
"2013": nil,
"green": nil,
"la": nil,
"london": nil,
"adam": nil,
"jnrsnrgaming": nil,
"jnrsnr": nil,
"TRUE": nil,
"junior senior gaming": nil,
"the kag3 gaming": nil,
"sonny daniel": nil,
"atlanta": nil,
"wordofgod": nil,
"2012": nil,
"Katrina kaif": nil,
"English": nil,
"Addon": nil,
"High": nil,
"To": nil,
"willie": nil,
"it": nil,
"trolling": nil,
"Bollywood news": nil,
"willie pelissier": nil,
"canada": nil,
"productions": nil,
"iMovie": nil,
"channel": nil,
"total": nil,
"trick": nil,
"the kag3": nil,
"Scurvin": nil,
"Free": nil,
"make": nil,
"The": nil,
"ops": nil,
"trevor": nil,
"kevin": nil,
"life is strange": nil,
"you": nil,
"kokesh": nil,
"the brotherhood of gaming": nil,
"Martinez ca": nil,
"real": nil,
"เกม": nil,
"sonny daniel vlogs": nil,
"China": nil,
"African American": nil,
"Slime": nil,
"America": nil,
"price": nil,
"michigan": nil,
"call": nil,
"truetotalempireinc": nil,
"Akshay Kumar": nil,
"Shield": nil,
"aventure": nil,
"until dawn": nil,
"camera": nil,
"florida": nil,
"with": nil,
"battlefield": nil,
"joe": nil,
"tom": nil,
"Atmospheric": nil,
"squad ops": nil,
"overview": nil,
"fist": nil,
"clan": nil,
"guy": nil,
"sonnydaniel": nil,
"Voice Over": nil,
"Test": nil,
"HouseParty": nil,
"ending": nil,
"my": nil,
"garena": nil,
"kag3 entertainment": nil,
"{5859dfec-026f-46ba-bea0-02bf43aa1a6f}": nil,
"shahrukh Khan": nil,
"Opening": nil,
"erik": nil,
"part 1": nil,
"UK": nil,
"dean": nil,
"hcg": nil,
"4k": nil,
"cheat": nil,
}
var mapAndReplace = map[string]string{
"nfl": Sports,
"Minecraft (Award-Winning Work)": Gaming,
"minecraft survival": Gaming,
"modded minecraft": Gaming,
"minecraft movie": Gaming,
"minecraft videos": Gaming,
}
var mapAndKeep = map[string]string{
"gameplay": Gaming,
"games": Gaming,
"News": News,
"game": Gaming,
"politics": News,
"PlayStation 4": Gaming,
"Sony Interactive Entertainment": Gaming,
"walkthrough": Gaming,
"pc": Technology,
"playthrough": Gaming,
"#PS4Live": Gaming,
"let's play": Gaming,
"democrat": News,
"liberal": News,
"progressive": News,
"government": News,
"play": Gaming,
"republican": News,
"conservative": News,
"commentary": News,
"political": News,
"CNN": News,
"tutorial": Learning,
"cbs": News,
"lets play": Gaming,
"MSNBC": News,
"bitcoin": Blockchain,
"minecraft": Gaming,
"twitch": Gaming,
"Video Game (Industry)": Gaming,
"ps4": Gaming,
"#PS4share": Gaming,
"how to": Learning,
"Fox news": News,
"fox": News,
"xbox": Gaming,
"nbc": News,
"Gaming": Gaming,
"comedy": Funny,
"video game": Gaming,
"car": Automotive,
"Education": Learning,
"POLITICS": News,
"Educational": Learning,
"cryptocurrency": Blockchain,
"News & Politics": News,
"fun": Funny,
"playstation": Gaming,
"CARS": Automotive,
"xbox one": Gaming,
"senate": News,
"steam": Gaming,
"4x4": Automotive,
"AUTOMOBILE": Automotive,
"fortnite": Gaming,
"autos": Automotive,
"Sport": Sports,
"crypto": Blockchain,
"Carros": Automotive,
"ethereum": Blockchain,
"Suv": Automotive,
"playstation 4": Gaming,
"TRUCK": Automotive,
"video games": Gaming,
"Funny": Funny,
"Garage": Automotive,
"congress": News,
"lets": Gaming,
"nintendo": Gaming,
"ps3": Gaming,
"auto": Automotive,
"house of representatives": News,
"fps": Gaming,
"rpg": Gaming,
"btc": Blockchain,
"video game (industry)": Gaming,
"multiplayer": Gaming,
"video news": News,
"animated news": News,
"android": Technology,
"computer": Technology,
"how": Learning,
"let's": Gaming,
"donald trump": News,
"Fortnite": Gaming,
"COMO": News,
"online": Technology,
"TRUMP": News,
"mods": Technology,
"tech": Technology,
"hilarious": Funny,
"dota 2": Gaming,
"crash": Automotive,
"PUTIN": News,
"dance": Art,
"money": Economics,
"news radio": News,
"lol": Funny,
"xbox 360": Gaming,
"SALMAN KHAN": Learning,
"airdrop": Blockchain,
"litecoin": Blockchain,
"Sony Computer Entertainment": Gaming,
"pc games": Gaming,
"eth": Blockchain,
"mining": Blockchain,
"ios": Technology,
"linux": Technology,
"sony": Technology,
"space": Nature,
"iphone": Technology,
"education": Learning,
"Gamer": Gaming,
"bitcoin news": Blockchain,
"call of duty": Gaming,
"pokemon": Gaming,
"League": Gaming,
"Let's Play": Gaming,
"Let's": Gaming,
"league of legends": Gaming,
"League Of Legends (Video Game)": Gaming,
"economy": Economics,
"cod": Gaming,
"ico": Blockchain,
"pubg": Gaming,
"gold": Economics,
"Xbox": Gaming,
"pvp": Gaming,
"videogame": Gaming,
"vegan": Food,
"dota": Gaming,
"Games": Gaming,
"let's play fr": Gaming,
"satire": Funny,
"mario": Gaming,
"pc game": Gaming,
"help": Learning,
"PS4": Gaming,
"crypto news": Blockchain,
"humor": Funny,
"let": Gaming,
"fallout 4": Gaming,
"Shooter Game (Media Genre)": Gaming,
"funny moments": Funny,
"Comedy": Funny,
"Playthrough": Gaming,
"gamer": Gaming,
"wii": Gaming,
"playing": Gaming,
"PC": Technology,
"computer game games": Gaming,
"Gameplay": Gaming,
"arcade": Gaming,
"token": Blockchain,
"federal reserve": Economics,
"Video Game Culture": Gaming,
"driving": Automotive,
"plants": Nature,
"trees": Nature,
"eggs": Nature,
"Xbox 360 (Video Game Platform)": Gaming,
"RPG": Gaming,
"design": Art,
"market": Economics,
"cars": Automotive,
"fruits": Food,
"flowers": Nature,
"ram": Technology,
"lessons": Learning,
"token free": Blockchain,
"vegán": Food,
"Game": Gaming,
"investing": Economics,
"samsung": Technology,
"racing": Sports,
"online learning": Learning,
"dollar": Economics,
"software": Technology,
"beach": Nature,
"gta": Gaming,
"windows": Technology,
"let’s play": Gaming,
"Minecraft": Gaming,
"race": Automotive,
"skyrim": Gaming,
"rainbow": Nature,
"health": Fitness,
"Action-adventure Game (Media Genre)": Gaming,
"programming": Technology,
"sony interactive entertainment": Gaming,
"dlc": Gaming,
"diy": Learning,
"bitcoin price": Blockchain,
"monetary reform": Economics,
"xbox360": Gaming,
"zelda": Gaming,
"Steam": Gaming,
"capcom": Gaming,
"economic": Economics,
"simulation": Technology,
"recession": Economics,
"fortnite battle royale": Gaming,
"altcoins": Blockchain,
"facebook": Technology,
"cryptocurrency news": Blockchain,
"singleplayer": Gaming,
"altcoin": Blockchain,
"pacman": Gaming,
"fashion": Art,
"gta 5": Gaming,
"Grand Theft Auto V": Gaming,
"valve": Gaming,
"Massively Multiplayer Online Role-playing Game (Video Game Genre)": Gaming,
"#gaming": Gaming,
"Bmw": Automotive,
"halo": Gaming,
"Hillary Clinton": News,
"trading": Economics,
"computer games": Gaming,
"vehicle": Automotive,
"desktop": Technology,
"coin": Blockchain,
"water": Nature,
"nintendo switch": Gaming,
"progressive talk": News,
"3d games": Gaming,
"animals": Nature,
"install": Technology,
"MOBA": Gaming,
"Ps2": Gaming,
"Role-playing Video Game (Media Genre)": Gaming,
"trump": News,
"ripple": Blockchain,
"parody": Funny,
"supercell": Gaming,
"dota2": Gaming,
"microsoft": Technology,
"sims": Gaming,
"barack obama": News,
"funny video": Funny,
"finance": Economics,
"floral": Nature,
"creative hairstyles": Art,
"food holidays": Food,
"colorful hair": Art,
"currency": Economics,
"free bitcoin": Blockchain,
"how-to": Learning,
"pc gaming": Gaming,
"Playstation 4": Gaming,
"twitter": Technology,
"battle royale": Gaming,
"rpg games": Gaming,
"sims 4": Gaming,
"Android": Technology,
"Benz": Automotive,
"silly": Funny,
"roblox": Gaming,
"Mw3": Gaming,
"humour": Funny,
"far cry 5": Gaming,
"Mobile": Technology,
"squad gameplay": Gaming,
"tf2": Gaming,
"Auto Show (Event)": Automotive,
"business": Economics,
"apple": Technology,
"Ps3": Gaming,
"Resident Evil": Gaming,
"iOS": Technology,
"Mercedes": Automotive,
"engine": Automotive,
"indie game": Gaming,
"overwatch": Gaming,
"brawl": Gaming,
"ltc": Blockchain,
"Call of Duty®: Black Ops III": Gaming,
"Donald Trump": News,
"dash": Blockchain,
"newscast": News,
"server": Technology,
"gardening": Nature,
"console": Gaming,
"mugen": Gaming,
"Play": Gaming,
"Nvidia": Technology,
"cool long hair": Art,
"howto": Learning,
"Mass effect": Gaming,
"mercedes": Automotive,
"ubisoft": Gaming,
"open source": Technology,
"Minecraft: PlayStation®4 Edition": Gaming,
"rts": Gaming,
"mmorpg": Gaming,
"first person shooter": Gaming,
"eos": Blockchain,
"motorcycle": Automotive,
"Action Role-playing Game (Video Game Genre)": Gaming,
"steemit": Blockchain,
"porsche": Automotive,
"sport": Sports,
"training": Sports,
"Apple": Technology,
"coinbase": Blockchain,
"Role-playing Game (Game Genre)": Gaming,
"sonic": Gaming,
"advice": Learning,
"videogames": Gaming,
"Simulator": Gaming,
"level": Gaming,
"apps": Technology,
"sims 3": Gaming,
"Satire": Funny,
"esports": Gaming,
"Minecraft (Video Game)": Gaming,
"simulator": Gaming,
"horrible gamers": Gaming,
"Bus": Automotive,
"moba": Gaming,
"football": Sports,
"cryptocurrencies": Blockchain,
"the game": Gaming,
"laptop": Technology,
"link": Gaming,
"Game Reviews": Gaming,
"sims 5": Gaming,
"snk": Gaming,
"online games": Gaming,
"squad game": Gaming,
"switch": Gaming,
"Lets Play": Gaming,
"racism": News,
"jeep": Automotive,
}
+18 -2
View File
@@ -1,6 +1,7 @@
package thumbs
import (
"google.golang.org/api/youtube/v3"
"io"
"net/http"
"os"
@@ -21,6 +22,7 @@ type thumbnailUploader struct {
}
const thumbnailPath = "/tmp/ytsync_thumbnails/"
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
func (u *thumbnailUploader) downloadThumbnail() error {
_ = os.Mkdir(thumbnailPath, 0750)
@@ -44,7 +46,7 @@ func (u *thumbnailUploader) downloadThumbnail() error {
}
func (u *thumbnailUploader) uploadThumbnail() error {
key := aws.String("/thumbnails/" + u.name)
key := &u.name
thumb, err := os.Open("/tmp/ytsync_thumbnails/" + u.name)
if err != nil {
return errors.Err(err)
@@ -62,8 +64,9 @@ func (u *thumbnailUploader) uploadThumbnail() error {
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
})
u.mirroredUrl = "https://thumbnails.lbry.com/" + u.name
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)
}
@@ -92,3 +95,16 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
return tu.mirroredUrl, nil
}
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
if thumbnails.Maxres != nil {
return thumbnails.Maxres
} else if thumbnails.High != nil {
return thumbnails.High
} else if thumbnails.Medium != nil {
return thumbnails.Medium
} else if thumbnails.Standard != nil {
return thumbnails.Standard
}
return thumbnails.Default
}