Compare commits

...
7 Commits
Author SHA1 Message Date
Niko Storni 52cd483f47 add single channelwide tag
fix bug
2019-06-10 21:37:13 +02:00
Niko Storni 79dd6d1458 add better handling for database mismatches
improve UTXOs management
fix bugs
2019-06-10 02:23:07 +02:00
Niko Storni 10ad46f464 various bug fixes 2019-06-06 23:25:31 +02:00
Niko Storni ccebab35da fix bugs 2019-06-06 17:26:10 +02:00
Niko Storni 8dce052fe6 change logic used to upgrade videos 2019-06-06 16:24:20 +02:00
Niko Storni 76e653fb9b add Fee support
lowercase all tags
refactor code
implement and test regexps for tags
2019-06-06 02:16:07 +02:00
Niko Storni cfe8ff0879 add flag to remove unpublished videos
refactor code
add workaround for SDK bug
add code to remove unpublished videos
fix alt downloader to only produce mp4
add missing height and width
add tags sorting
add a few channel wide tags
2019-06-04 22:21:44 +02:00
10 changed files with 388 additions and 150 deletions

No files matched your search

+1 -1
View File
@@ -15,7 +15,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.7
github.com/lbryio/lbry.go v1.0.9
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
+6 -2
View File
@@ -18,6 +18,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -59,6 +60,7 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -127,10 +129,11 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v1.0.7 h1:CO9wnH/grsrX1O3YpeSBpLVPtc3lOOVTvZCcPFZ2Os0=
github.com/lbryio/lbry.go v1.0.7/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.9 h1:SAqopNiISazYuF38quA3WEfqOCzFtUed6QfCkAYQoJo=
github.com/lbryio/lbry.go v1.0.9/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
@@ -194,6 +197,7 @@ github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+6
View File
@@ -36,6 +36,8 @@ var (
videosLimit int
maxVideoSize int
maxVideoLength float64
removeDBUnpublished bool
upgradeMetadata bool
)
func main() {
@@ -56,6 +58,8 @@ func main() {
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
@@ -188,6 +192,8 @@ func ytSync(cmd *cobra.Command, args []string) {
syncProperties,
apiConfig,
maxVideoLength,
removeDBUnpublished,
upgradeMetadata,
)
err := sm.Start()
if err != nil {
+10 -3
View File
@@ -2,8 +2,6 @@ package manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"strings"
"syscall"
"time"
@@ -13,6 +11,9 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
)
@@ -39,12 +40,14 @@ type SyncManager struct {
singleRun bool
syncProperties *sdk.SyncProperties
apiConfig *sdk.APIConfig
removeDBUnpublished bool
upgradeMetadata bool
}
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
return &SyncManager{
stopOnError: stopOnError,
maxTries: maxTries,
@@ -68,6 +71,8 @@ func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool
singleRun: singleRun,
syncProperties: syncProperties,
apiConfig: apiConfig,
removeDBUnpublished: removeDBUnpublished,
upgradeMetadata: upgradeMetadata,
}
}
@@ -131,6 +136,7 @@ func (s *SyncManager) Start() error {
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: channels[0].Fee,
}
shouldInterruptLoop = true
} else {
@@ -165,6 +171,7 @@ func (s *SyncManager) Start() error {
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: c.Fee,
})
}
}
+18 -22
View File
@@ -2,6 +2,7 @@ package manager
import (
"fmt"
"math"
"net/http"
"os"
"strconv"
@@ -21,7 +22,7 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 3
const minimumRefillAmount = 4
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
@@ -84,7 +85,7 @@ func (s *Sync) walletSetup() error {
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if s.Manager.syncStatus == StatusPendingUpgrade {
if s.Manager.upgradeMetadata {
videosToUpgrade := 0
for _, v := range s.syncedVideos {
if v.Published && v.MetadataVersion < 2 {
@@ -189,9 +190,10 @@ func (s *Sync) ensureEnoughUTXOs() error {
broadcastFee := 0.01
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
desiredUTXOCount := uint64(math.Floor((balanceAmount - broadcastFee) / 0.1))
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, desiredUTXOCount)
if err != nil {
return err
} else if prefillTx == nil {
@@ -355,34 +357,28 @@ func (s *Sync) ensureChannelOwnership() error {
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
}
var c *jsonrpc.TransactionSummary
claimCreateOptions := jsonrpc.ClaimCreateOptions{
Title: &channelInfo.Title,
Description: &channelInfo.Description,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
}
if channelUsesOldMetadata {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
CoverURL: bannerURL,
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
},
})
} else {
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
CoverURL: bannerURL,
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
})
}
+81 -34
View File
@@ -74,18 +74,17 @@ type Sync struct {
AwsS3Secret string
AwsS3Region string
AwsS3Bucket string
daemon *jsonrpc.Client
claimAddress string
videoDirectory string
syncedVideosMux *sync.RWMutex
syncedVideos map[string]sdk.SyncedVideo
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.Mutex
queue chan video
Fee *sdk.Fee
daemon *jsonrpc.Client
claimAddress string
videoDirectory string
syncedVideosMux *sync.RWMutex
syncedVideos map[string]sdk.SyncedVideo
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.Mutex
queue chan video
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
@@ -445,8 +444,10 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
}
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err error) {
//additionally it removes all entries in the database indicating that a video is published when it's actually not
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
count := 0
videoIDMap := make(map[string]string, len(claims))
for _, c := range claims {
if !isYtsyncClaim(c) {
continue
@@ -455,22 +456,60 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
//check if claimID is in remote db
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
pv, ok := s.syncedVideos[videoID]
if !ok || pv.ClaimName != c.Name {
fixed++
log.Debugf("adding %s to the database", c.Name)
size, err := c.GetStreamSizeByMagic()
videoIDMap[videoID] = c.ClaimID
pv, claimInDatabase := s.syncedVideos[videoID]
claimMetadataVersion := uint(1)
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
claimMetadataVersion = 2
}
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
claimMarkedUnpublished := claimInDatabase && !pv.Published
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
}
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
}
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
}
if claimMarkedUnpublished {
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
}
if !claimInDatabase {
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
claimSize, err := c.GetStreamSizeByMagic()
if err != nil {
size = 0
claimSize = 0
}
metadataVersion := uint(1)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
fixed++
log.Debugf("updating %s in the database", videoID)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion)
if err != nil {
return count, fixed, err
return count, fixed, 0, err
}
}
}
return count, fixed, nil
idsToRemove := make([]string, 0, len(videoIDMap))
for vID, sv := range s.syncedVideos {
_, ok := videoIDMap[vID]
if !ok && sv.Published {
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
idsToRemove = append(idsToRemove, vID)
}
}
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
if err != nil {
return count, fixed, 0, err
}
}
return count, fixed, len(idsToRemove), nil
}
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
@@ -516,16 +555,21 @@ func (s *Sync) doSync() error {
}
}
pubsOnWallet, nFixed, err := s.updateRemoteDB(allClaims)
pubsOnWallet, nFixed, nRemoved, err := s.updateRemoteDB(allClaims)
if err != nil {
return errors.Prefix("error counting claims", err)
return errors.Prefix("error updating remote database", err)
}
if nFixed > 0 {
if nFixed > 0 || nRemoved > 0 {
err := s.setStatusSyncing()
if err != nil {
return err
}
SendInfoToSlack("%d claims were not on the remote database and were fixed", nFixed)
if nFixed > 0 {
SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
}
if nRemoved > 0 {
SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
}
}
pubsOnDB := 0
for _, sv := range s.syncedVideos {
@@ -555,7 +599,7 @@ func (s *Sync) doSync() error {
}
if s.LbryChannelName == "@UCBerkeley" {
err = errors.Err("UCB is not supported on this version of YTSYNC")
err = errors.Err("UCB is not supported in this version of YTSYNC")
} else {
err = s.enqueueYoutubeVideos()
}
@@ -648,6 +692,8 @@ func (s *Sync) startWorker(workerNum int) {
SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
break
}
} else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") {
time.Sleep(5 * time.Second)
}
log.Println("Retrying")
continue
@@ -667,7 +713,7 @@ func (s *Sync) startWorker(workerNum int) {
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 0)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@@ -805,8 +851,9 @@ func (s *Sync) processVideo(v video) (err error) {
s.syncedVideosMux.RLock()
sv, ok := s.syncedVideos[v.ID()]
s.syncedVideosMux.RUnlock()
newMetadataVersion := int8(2)
alreadyPublished := ok && sv.Published
isUpgradeSync := s.Manager.syncStatus == StatusPendingUpgrade
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
neverRetryFailures := []string{
"Error extracting sts from embedded url response",
@@ -822,17 +869,16 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
if alreadyPublished && !isUpgradeSync {
if alreadyPublished && !videoRequiresUpgrade {
log.Println(v.ID() + " already published")
return nil
}
newMetadataVersion := int8(2)
if ok && sv.MetadataVersion >= newMetadataVersion {
log.Println(v.ID() + " upgraded to the new metadata")
return nil
}
if v.PlaylistPosition() > s.Manager.videosLimit {
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
log.Println(v.ID() + " is old: skipping")
return nil
}
@@ -847,9 +893,10 @@ func (s *Sync) processVideo(v video) (err error) {
MaxVideoSize: s.Manager.maxVideoSize,
Namer: s.namer,
MaxVideoLength: s.Manager.maxVideoLength,
Fee: s.Fee,
}
summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync)
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade)
if err != nil {
return err
}
+39 -7
View File
@@ -34,16 +34,17 @@ type SyncProperties struct {
YoutubeChannelID string
}
type Fee struct {
Amount string `json:"amount"`
Address string `json:"address"`
Currency string `json:"currency"`
}
type YoutubeChannel struct {
ChannelId string `json:"channel_id"`
TotalVideos uint `json:"total_videos"`
DesiredChannelName string `json:"desired_channel_name"`
Fee *struct {
Amount string `json:"amount"`
Address string `json:"address"`
Currency string `json:"currency"`
} `json:"fee"`
ChannelClaimID string `json:"channel_claim_id"`
Fee *Fee `json:"fee"`
ChannelClaimID string `json:"channel_claim_id"`
}
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
@@ -167,6 +168,35 @@ const (
VideoStatusFailed = "failed"
)
func (a *APIConfig) DeleteVideos(videos []string) error {
endpoint := a.ApiURL + "/yt/video_delete"
videoIDs := strings.Join(videos, ",")
vals := url.Values{
"video_ids": {videoIDs},
"auth_token": {a.ApiToken},
}
res, _ := http.PostForm(endpoint, vals)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data null.String `json:"data"`
}
err := json.Unmarshal(body, &response)
if err != nil {
return errors.Err(err)
}
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
if !response.Data.IsNull() && response.Data.String == "ok" {
return nil
}
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
endpoint := a.ApiURL + "/yt/video_status"
@@ -184,7 +214,9 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
if metadataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
}
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
}
+97 -71
View File
@@ -24,6 +24,7 @@ import (
"github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
)
@@ -50,38 +51,38 @@ type YoutubeVideo struct {
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "Film & Animation",
"2": "Autos & Vehicles",
"10": "Music",
"15": "Pets & Animals",
"17": "Sports",
"18": "Short Movies",
"19": "Travel & Events",
"20": "Gaming",
"21": "Videoblogging",
"22": "People & Blogs",
"23": "Comedy",
"24": "Entertainment",
"25": "News & Politics",
"26": "Howto & Style",
"27": "Education",
"28": "Science & Technology",
"29": "Nonprofits & Activism",
"30": "Movies",
"31": "Anime/Animation",
"32": "Action/Adventure",
"33": "Classics",
"34": "Comedy",
"35": "Documentary",
"36": "Drama",
"37": "Family",
"38": "Foreign",
"39": "Horror",
"40": "Sci-Fi/Fantasy",
"41": "Thriller",
"42": "Shorts",
"43": "Shows",
"44": "Trailers",
"1": "film & animation",
"2": "autos & vehicles",
"10": "music",
"15": "pets & animals",
"17": "sports",
"18": "short movies",
"19": "travel & events",
"20": "gaming",
"21": "videoblogging",
"22": "people & blogs",
"23": "comedy",
"24": "entertainment",
"25": "news & politics",
"26": "howto & style",
"27": "education",
"28": "science & technology",
"29": "nonprofits & activism",
"30": "movies",
"31": "anime/animation",
"32": "action/adventure",
"33": "classics",
"34": "comedy",
"35": "documentary",
"36": "drama",
"37": "family",
"38": "foreign",
"39": "horror",
"40": "sci-fi/fantasy",
"41": "thriller",
"42": "shorts",
"43": "shows",
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
@@ -174,8 +175,11 @@ func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+bestaudio/best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"))
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4")
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
@@ -316,19 +320,33 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
return err
}
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, namer *namer.Namer) (*SyncSummary, error) {
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
languages, locations, tags := v.getMetadata()
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
if err != nil {
return nil, errors.Err(err)
}
fee = &jsonrpc.Fee{
FeeAddress: &params.Fee.Address,
FeeAmount: feeAmount,
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
}
}
options := jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: v.title,
Description: v.getAbbrevDescription(),
ClaimAddress: &claimAddress,
Title: &v.title,
Description: util.PtrToString(v.getAbbrevDescription()),
ClaimAddress: &params.ClaimAddress,
Languages: languages,
ThumbnailURL: &v.thumbnailURL,
Tags: tags,
Locations: locations,
},
Fee: fee,
License: util.PtrToString("Copyrighted (contact publisher)"),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
ChannelID: &v.lbryChannelID,
@@ -337,7 +355,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
if err != nil {
return nil, err
}
return publishAndRetryExistingNames(daemon, v.title, downloadPath, amount, options, namer)
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
}
func (v *YoutubeVideo) Size() *int64 {
@@ -351,6 +369,7 @@ type SyncParams struct {
MaxVideoSize int
Namer *namer.Namer
MaxVideoLength float64
Fee *sdk.Fee
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
@@ -382,7 +401,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
}
log.Debugln("Created thumbnail for " + v.id)
summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.Namer)
summary, err := v.publish(daemon, params)
//delete the video in all cases (and ignore the error)
_ = v.delete()
@@ -452,19 +471,38 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err("the video must be republished as we can't get the right size")
}
}
v.size = util.PtrToInt64(int64(videoSize))
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
if err != nil {
return nil, errors.Err(err)
}
fee = &jsonrpc.Fee{
FeeAddress: &params.Fee.Address,
FeeAmount: feeAmount,
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
}
}
streamCreateOptions := &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Tags: tags,
ThumbnailURL: &thumbnailURL,
Languages: languages,
Locations: locations,
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),
ChannelID: &v.lbryChannelID,
Height: util.PtrToUint(720),
Width: util.PtrToUint(1280),
Fee: fee,
}
if v.mocked {
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Tags: tags,
ThumbnailURL: &thumbnailURL,
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),
ChannelID: &v.lbryChannelID,
},
FileSize: &videoSize,
StreamCreateOptions: streamCreateOptions,
FileSize: &videoSize,
})
if err != nil {
return nil, err
@@ -481,28 +519,16 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err(err)
}
streamCreateOptions.ClaimCreateOptions.Title = &v.title
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
ClearLanguages: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearTags: util.PtrToBool(true),
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: v.title,
Description: v.getAbbrevDescription(),
Tags: tags,
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),
VideoHeight: util.PtrToUint(720),
VideoWidth: util.PtrToUint(1280),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &v.lbryChannelID,
},
FileSize: &videoSize,
ClearLanguages: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearTags: util.PtrToBool(true),
StreamCreateOptions: streamCreateOptions,
FileSize: &videoSize,
})
if err != nil {
return nil, err
+32 -10
View File
@@ -1,7 +1,11 @@
package tagsManager
import (
"regexp"
"sort"
"strings"
log "github.com/sirupsen/logrus"
)
const (
@@ -60,6 +64,8 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
}
}
sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags))
sort.Strings(curatedTags)
sort.Strings(originalTags)
sanitizedTags = append(sanitizedTags, curatedTags...)
sanitizedTags = append(sanitizedTags, originalTags...)
return sanitizedTags, nil
@@ -67,17 +73,23 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
func normalizeTag(t string) (string, error) {
t = strings.ToLower(t)
multipleSpaces := regexp.MustCompile(`\s{2,}`)
leadingAndTrailingSpaces := regexp.MustCompile(`^\s+|\s$`)
hashTags := regexp.MustCompile(`(#\d+\s)|#+`)
inParenthesis := regexp.MustCompile(`\([^\)]+\)`)
weirdChars := regexp.MustCompile(`[^-\w'& +\/A-Za-zÀ-ÖØ-öø-ÿ]`)
startsOrEndsInWeirdChars := regexp.MustCompile(`^[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+|[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+$`)
t = hashTags.ReplaceAllString(t, "")
t = inParenthesis.ReplaceAllString(t, " ")
t = startsOrEndsInWeirdChars.ReplaceAllString(t, "")
t = multipleSpaces.ReplaceAllString(t, " ")
t = leadingAndTrailingSpaces.ReplaceAllString(t, "")
if weirdChars.MatchString(t) {
log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
return "", nil
}
return t, nil
//r, err := regexp.Compile("/\\([^\\)]+\\)/g")
//if err != nil {
// return "", errors.Err(err)
//}
//r2, err := regexp.Compile("/[^\\w-'& \+]/g")
//if err != nil {
// return "", errors.Err(err)
//}
//t = r.ReplaceAllString(t, "")
//return r2.ReplaceAllString(t, ""), nil
}
type tagsSanitizer struct {
@@ -130,11 +142,21 @@ func (ts *tagsSanitizer) add() {
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
DavidPakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
Juggling: {"juggling", "circus arts", "malabares"},
JustJuggling: {"juggling", "circus arts", "malabares"},
JordanBPeterson: {"postmodernism", "psychology", "news"},
DavidPakman: {"news"},
TheLinuxGamer: {"gaming", "technology", "linux"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,
+98
View File
@@ -1,6 +1,7 @@
package tagsManager
import (
"fmt"
"testing"
)
@@ -44,3 +45,100 @@ outer:
}
}
func TestNormalizeTag(t *testing.T) {
tags := []string{
"blockchain",
"Switzerland",
"news ",
" science & Technology ",
"economics",
"experiments",
"this",
"in it",
"will build the (WOOPS)",
"~has",
"crypto",
"trump",
"wall",
"expensive",
"!currency",
" a lot of ",
"#",
"#whatever",
"#123",
"#123 Something else",
"#123aaa",
"!asdasd",
"CASA BLANCA",
"wwe 2k18 Elimination chamber!",
"pero'",
"però",
"è proprio",
"Ep 29",
"sctest29 Keddr",
"mortal kombat 11 shang tsung",
"!asdasd!",
}
normalizedTags := make([]string, 0, len(tags))
for _, tag := range tags {
got, err := normalizeTag(tag)
if err != nil {
t.Error(err)
return
}
if got != "" {
normalizedTags = append(normalizedTags, got)
}
fmt.Printf("Got tag: '%s'\n", got)
}
expected := []string{
"blockchain",
"switzerland",
"news",
"science & technology",
"economics",
"experiments",
"this",
"in it",
"will build the",
"has",
"crypto",
"trump",
"wall",
"expensive",
"currency",
"a lot of",
"whatever",
"123",
"something else",
"123aaa",
"asdasd",
"casa blanca",
"wwe 2k18 elimination chamber",
"pero",
"però",
"è proprio",
"ep 29",
"sctest29 keddr",
"mortal kombat 11 shang tsung",
"asdasd",
}
if !Equal(normalizedTags, expected) {
t.Error("result not as expected")
return
}
}
func Equal(a, b []string) bool {
if len(a) != len(b) {
fmt.Printf("expected length %d but got %d", len(b), len(a))
return false
}
for i, v := range a {
if v != b[i] {
fmt.Printf("expected %s but bot %s\n", b[i], v)
return false
}
}
return true
}