Compare commits

..
Author SHA1 Message Date
Niko Storni 6668840b11 fix description not containing youtube link
add workaround to SDK bug not spending the right UTXOs
fix bug in assignments
2019-06-13 19:33:58 +02:00
Niko Storni e893f68bc1 fix queue pulling
fix address reuse
2019-06-12 22:42:42 +02:00
Niko Storni 8da631eac3 add a limit to UTXOs generation
fix broadcastFee
fix spelling
2019-06-12 05:33:13 +02:00
Niko Storni 1729af0ae5 fix UTXOs management while publishing
fix account_set
improve logging
2019-06-12 05:25:01 +02:00
Niko Storni dce71d2451 add adjustments to video tracking
reduce minimum refill amount (min account balance supersedes)
revert account_fund change as --everything isn't supported with outputs
fix bugs
2019-06-12 04:44:56 +02:00
Niko Storni f4e75cf221 rewrite account funding algorithm
lock publishing when UTXOs management is in progress
spend everything when funding UTXOs
update lbry.go
2019-06-12 03:17:59 +02:00
Niko Storni ae3dad944d add blocking support for publishing and upgrading
add channel wide mappings
2019-06-11 19:18:39 +02:00
Niko Storni f7f0231c80 add VideoUpgradeFailed staus 2019-06-10 21:59:42 +02:00
Niko Storni 0c42b96332 remove unused script
update readme
2019-06-10 21:43:10 +02:00
Niko Storni 52cd483f47 add single channelwide tag
fix bug
2019-06-10 21:37:13 +02:00
Niko Storni 79dd6d1458 add better handling for database mismatches
improve UTXOs management
fix bugs
2019-06-10 02:23:07 +02:00
Niko Storni 10ad46f464 various bug fixes 2019-06-06 23:25:31 +02:00
11 changed files with 591 additions and 282 deletions

No files matched your search

+3 -1
View File
@@ -55,7 +55,7 @@ Usage:
Flags:
--after int Specify from when to pull jobs [Unix time](Default: 0)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1582834707)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
--channelID string If specified, only this channel will be synced.
--concurrent-jobs int how many jobs to process concurrently (default 1)
-h, --help help for ytsync
@@ -63,12 +63,14 @@ Flags:
--max-length float Maximum video length to process (in hours) (default 2)
--max-size int Maximum video size to process (in MB) (default 2048)
--max-tries int Number of times to try a publish that fails (default 3)
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
--run-once Whether the process should be stopped after one cycle or not
--skip-space-check Do not perform free space check on startup
--status string Specify which queue to pull from. Overrides --update
--stop-on-error If a publish fails, stop all publishing and exit
--takeover-existing-channel If channel exists and we don't own it, take over the channel
--update Update previously synced channels instead of syncing new ones
--upgrade-metadata Upgrade videos if they're on the old metadata version
--videos-limit int how many videos to process per channel (default 1000)
```
+1 -1
View File
@@ -15,7 +15,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.9
github.com/lbryio/lbry.go v1.0.14
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
+2 -2
View File
@@ -129,8 +129,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v1.0.9 h1:SAqopNiISazYuF38quA3WEfqOCzFtUed6QfCkAYQoJo=
github.com/lbryio/lbry.go v1.0.9/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.14 h1:lpaO96YyP3d2RJzgl1WFkcyS15/ROd04OV3S1E5Av8E=
github.com/lbryio/lbry.go v1.0.14/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
+15 -9
View File
@@ -2,8 +2,6 @@ package manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"strings"
"syscall"
"time"
@@ -13,6 +11,9 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
)
@@ -90,9 +91,10 @@ const (
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUnpublished = "unpublished"
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
)
func (s *SyncManager) Start() error {
@@ -140,6 +142,7 @@ func (s *SyncManager) Start() error {
shouldInterruptLoop = true
} else {
var queuesToSync []string
//TODO: implement scrambling to avoid starvation of queues
if s.syncStatus != "" {
queuesToSync = append(queuesToSync, s.syncStatus)
} else if s.syncUpdate {
@@ -152,7 +155,9 @@ func (s *SyncManager) Start() error {
if err != nil {
return err
}
for _, c := range channels {
log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
if len(channels) > 0 {
c := channels[0]
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: c.ChannelId,
@@ -172,6 +177,7 @@ func (s *SyncManager) Start() error {
namer: namer.NewNamer(),
Fee: c.Fee,
})
break
}
}
}
@@ -179,9 +185,9 @@ func (s *SyncManager) Start() error {
log.Infoln("No channels to sync. Pausing 5 minutes!")
time.Sleep(5 * time.Minute)
}
for i, sync := range syncs {
for _, sync := range syncs {
shouldNotCount := false
SendInfoToSlack("Syncing %s (%s) to LBRY! (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
err := sync.FullCycle()
if err != nil {
fatalErrors := []string{
@@ -201,7 +207,7 @@ func (s *SyncManager) Start() error {
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
}
}
SendInfoToSlack("Syncing %s (%s) reached an end. (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
if !shouldNotCount {
syncCount++
}
+44 -41
View File
@@ -2,6 +2,7 @@ package manager
import (
"fmt"
"math"
"net/http"
"os"
"strconv"
@@ -21,8 +22,6 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 3
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
if err != nil {
@@ -34,7 +33,8 @@ func (s *Sync) enableAddressReuse() error {
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: 1000,
ChangeMaxUses: util.PtrToInt(1000),
ReceivingMaxUses: util.PtrToInt(100),
})
if err != nil {
return errors.Err(err)
@@ -43,7 +43,7 @@ func (s *Sync) enableAddressReuse() error {
return nil
}
func (s *Sync) walletSetup() error {
//prevent unnecessary concurrent execution
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
s.walletMux.Lock()
defer s.walletMux.Unlock()
err := s.ensureChannelOwnership()
@@ -67,51 +67,51 @@ func (s *Sync) walletSetup() error {
if err != nil {
return err
}
numOnSource := int(n)
videosOnYoutube := int(n)
log.Debugf("Source channel has %d videos", numOnSource)
if numOnSource == 0 {
log.Debugf("Source channel has %d videos", videosOnYoutube)
if videosOnYoutube == 0 {
return nil
}
s.syncedVideosMux.RLock()
numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones...
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d videos", numPublished)
if numOnSource-numPublished > s.Manager.videosLimit {
numOnSource = s.Manager.videosLimit
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if s.Manager.upgradeMetadata {
videosToUpgrade := 0
for _, v := range s.syncedVideos {
if v.Published && v.MetadataVersion < 2 {
videosToUpgrade++
publishedCount := 0
notUpgradedCount := 0
failedCount := 0
for _, sv := range s.syncedVideos {
if sv.Published {
publishedCount++
if sv.MetadataVersion < 2 {
notUpgradedCount++
}
} else {
failedCount++
}
minBalance += float64(videosToUpgrade) * 0.001
}
if numPublished > numOnSource && balance < minimumRefillAmount {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = minimumRefillAmount
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d videos", publishedCount)
if videosOnYoutube > s.Manager.videosLimit {
videosOnYoutube = s.Manager.videosLimit
}
unallocatedVideos := videosOnYoutube - publishedCount
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
if s.Manager.upgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
}
refillAmount := 0.0
if balance < requiredBalance || balance < minimumAccountBalance {
refillAmount = math.Max(requiredBalance-requiredBalance, minimumRefillAmount)
}
amountToAdd := minBalance - balance
if s.Refill > 0 {
if amountToAdd < 0 {
amountToAdd = float64(s.Refill)
} else {
amountToAdd += float64(s.Refill)
}
refillAmount += float64(s.Refill)
}
if amountToAdd > 0 {
if amountToAdd < minimumRefillAmount {
amountToAdd = minimumRefillAmount
}
err := s.addCredits(amountToAdd)
if refillAmount > 0 {
err := s.addCredits(refillAmount)
if err != nil {
return errors.Err(err)
}
@@ -186,12 +186,15 @@ func (s *Sync) ensureEnoughUTXOs() error {
if err != nil {
return errors.Err(err)
}
broadcastFee := 0.01
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
maxUTXOs := uint64(500)
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
if desiredUTXOCount > maxUTXOs {
desiredUTXOCount = maxUTXOs
}
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
broadcastFee := 0.1
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
if err != nil {
return err
} else if prefillTx == nil {
+86 -52
View File
@@ -37,9 +37,12 @@ import (
)
const (
channelClaimAmount = 0.01
publishAmount = 0.01
maxReasonLength = 500
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.1
minimumAccountBalance = 4.0
minimumRefillAmount = 1
publishAmount = 0.01
maxReasonLength = 500
)
type video interface {
@@ -48,7 +51,7 @@ type video interface {
IDAndNum() string
PlaylistPosition() int
PublishedAt() time.Time
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool) (*sources.SyncSummary, error)
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
}
// sorting videos
@@ -83,17 +86,21 @@ type Sync struct {
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.Mutex
walletMux *sync.RWMutex
queue chan video
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
s.syncedVideosMux.Lock()
defer s.syncedVideosMux.Unlock()
s.syncedVideos[videoID] = sdk.SyncedVideo{
VideoID: videoID,
Published: published,
FailureReason: failureReason,
VideoID: videoID,
Published: published,
FailureReason: failureReason,
ClaimID: claimID,
ClaimName: claimName,
MetadataVersion: metadataVersion,
Size: size,
}
}
@@ -254,7 +261,7 @@ func (s *Sync) FullCycle() (e error) {
s.setExceptions()
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.Mutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stop.New()
s.queue = make(chan video)
interruptChan := make(chan os.Signal, 1)
@@ -444,6 +451,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
}
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
//additionally it removes all entries in the database indicating that a video is published when it's actually not
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
count := 0
videoIDMap := make(map[string]string, len(claims))
@@ -456,20 +464,39 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
videoIDMap[videoID] = c.ClaimID
pv, ok := s.syncedVideos[videoID]
if !ok || pv.ClaimName != c.Name {
fixed++
log.Debugf("adding %s to the database", c.Name)
size, err := c.GetStreamSizeByMagic()
pv, claimInDatabase := s.syncedVideos[videoID]
claimMetadataVersion := uint(1)
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
claimMetadataVersion = 2
}
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
claimMarkedUnpublished := claimInDatabase && !pv.Published
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
}
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
}
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
}
if claimMarkedUnpublished {
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
}
if !claimInDatabase {
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
claimSize, err := c.GetStreamSizeByMagic()
if err != nil {
size = 0
claimSize = 0
}
metadataVersion := uint(1)
claimIsUpgraded := strings.Contains(tn, thumbs.ThumbnailEndpoint)
if claimIsUpgraded {
metadataVersion = 2
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
fixed++
log.Debugf("updating %s in the database", videoID)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion)
if err != nil {
return count, fixed, 0, err
}
@@ -483,15 +510,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
idsToRemove = append(idsToRemove, vID)
}
}
removeCount := 0
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
if err != nil {
return count, fixed, 0, err
}
removeCount++
}
return count, fixed, removeCount, nil
return count, fixed, len(idsToRemove), nil
}
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
@@ -547,7 +572,7 @@ func (s *Sync) doSync() error {
return err
}
if nFixed > 0 {
SendInfoToSlack("%d claims were not on the remote database and were fixed", nFixed)
SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
}
if nRemoved > 0 {
SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
@@ -581,7 +606,7 @@ func (s *Sync) doSync() error {
}
if s.LbryChannelName == "@UCBerkeley" {
err = errors.Err("UCB is not supported on this version of YTSYNC")
err = errors.Err("UCB is not supported in this version of YTSYNC")
} else {
err = s.enqueueYoutubeVideos()
}
@@ -620,7 +645,7 @@ func (s *Sync) startWorker(workerNum int) {
err := s.processVideo(v)
if err != nil {
logMsg := "error processing video: " + err.Error()
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
log.Errorln(logMsg)
fatalErrors := []string{
":5279: read: connection reset by peer",
@@ -651,24 +676,27 @@ func (s *Sync) startWorker(workerNum int) {
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
log.Println("waiting for a block before retrying")
err := s.waitForNewBlock()
if err != nil {
s.grp.Stop()
SendErrorToSlack("something went wrong while waiting for a block: %v", err)
break
}
} else if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
} else if tryCount < s.MaxTries {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
log.Println("waiting for a block before retrying")
err = s.waitForNewBlock()
if err != nil {
s.grp.Stop()
SendErrorToSlack("something went wrong while waiting for a block: %v", err)
break
}
} else if util.SubstringInSlice(err.Error(), []string{
if util.SubstringInSlice(err.Error(), []string{
"Not enough funds to cover this transaction",
"failed: Not enough funds",
"Error in daemon: Insufficient funds, please deposit additional LBC"}) {
log.Println("refilling addresses before retrying")
err = s.walletSetup()
"Error in daemon: Insufficient funds, please deposit additional LBC",
// "txn-mempool-conflict", //TODO: uncomment the two lines when the SDK will start spending confirmed UTXOs before failing
//"too-long-mempool-chain",
}) {
log.Println("checking funds and UTXOs before retrying...")
err := s.walletSetup()
if err != nil {
s.grp.Stop()
SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
@@ -685,17 +713,24 @@ func (s *Sync) startWorker(workerNum int) {
existingClaim, ok := s.syncedVideos[v.ID()]
existingClaimID := ""
existingClaimName := ""
existingClaimSize := v.Size()
existingClaimSize := int64(0)
if v.Size() != nil {
existingClaimSize = *v.Size()
}
if ok {
existingClaimID = existingClaim.ClaimID
existingClaimName = existingClaim.ClaimName
if existingClaim.Size > 0 {
existingClaimSize = &existingClaim.Size
existingClaimSize = existingClaim.Size
}
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
videoStatus := VideoStatusFailed
if strings.Contains(err.Error(), "upgrade failed") {
videoStatus = VideoStatusUpgradeFailed
} else {
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@@ -835,7 +870,7 @@ func (s *Sync) processVideo(v video) (err error) {
s.syncedVideosMux.RUnlock()
newMetadataVersion := int8(2)
alreadyPublished := ok && sv.Published
videoRequiresUpgrade := s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
neverRetryFailures := []string{
"Error extracting sts from embedded url response",
@@ -860,7 +895,7 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
if v.PlaylistPosition() > s.Manager.videosLimit {
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
log.Println(v.ID() + " is old: skipping")
return nil
}
@@ -878,18 +913,17 @@ func (s *Sync) processVideo(v video) (err error) {
Fee: s.Fee,
}
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade)
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
if err != nil {
return err
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName)
return nil
}
+8 -5
View File
@@ -164,8 +164,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusPublished = "published"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusFailed = "failed"
)
func (a *APIConfig) DeleteVideos(videos []string) error {
@@ -207,14 +208,16 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
"status": {status},
"auth_token": {a.ApiToken},
}
if status == VideoStatusPublished {
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
if claimID == "" || claimName == "" {
return errors.Err("claimID or claimName missing")
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
if metadataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
}
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
}
+4 -1
View File
@@ -2,6 +2,7 @@ package sources
import (
"strings"
"sync"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/ytsync/namer"
@@ -12,7 +13,9 @@ type SyncSummary struct {
ClaimName string
}
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer) (*SyncSummary, error) {
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer, walletLock *sync.RWMutex) (*SyncSummary, error) {
walletLock.RLock()
defer walletLock.RUnlock()
for {
name := namer.GetNextName(title)
response, err := daemon.StreamCreate(name, filename, amount, options)
+20 -8
View File
@@ -10,6 +10,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
@@ -46,6 +47,7 @@ type YoutubeVideo struct {
thumbnailURL string
lbryChannelID string
mocked bool
walletLock *sync.RWMutex
}
const reflectorURL = "http://blobs.lbry.io/"
@@ -160,25 +162,25 @@ func (v *YoutubeVideo) getFullPath() string {
func (v *YoutubeVideo) getAbbrevDescription() string {
maxLines := 10
description := strings.TrimSpace(v.description)
if strings.Count(description, "\n") < maxLines {
return description
}
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
if v.lbryChannelID == khanAcademyClaimID {
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
}
if strings.Count(description, "\n") < maxLines {
return description + "\n..." + additionalDescription
}
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4")
"mp4",
"https://www.youtube.com/watch?v="+v.ID())
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
@@ -187,6 +189,12 @@ func (v *YoutubeVideo) fallbackDownload() error {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
return errors.Err(err)
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
@@ -355,7 +363,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
if err != nil {
return nil, err
}
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer, v.walletLock)
}
func (v *YoutubeVideo) Size() *int64 {
@@ -372,13 +380,14 @@ type SyncParams struct {
Fee *sdk.Fee
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
if reprocess && existingVideoData != nil && existingVideoData.Published {
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, err
return summary, errors.Prefix("upgrade failed", err)
}
return v.downloadAndPublish(daemon, params)
}
@@ -471,6 +480,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err("the video must be republished as we can't get the right size")
}
}
v.size = util.PtrToInt64(int64(videoSize))
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
@@ -498,6 +508,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
Fee: fee,
}
v.walletLock.RLock()
defer v.walletLock.RUnlock()
if v.mocked {
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: streamCreateOptions,
-152
View File
@@ -1,152 +0,0 @@
import os
import sys
from decimal import Decimal
from bitcoinrpc.authproxy import AuthServiceProxy
from lbryum.wallet import Wallet, WalletStorage
from lbryum.commands import known_commands, Commands
from lbryum.simple_config import SimpleConfig
from lbryum.blockchain import get_blockchain
from lbryum.network import Network
def get_lbrycrdd_connection_string(wallet_conf):
settings = {"username": "rpcuser",
"password": "rpcpassword",
"rpc_port": 9245}
if wallet_conf and os.path.exists(wallet_conf):
with open(wallet_conf, "r") as conf:
conf_lines = conf.readlines()
for l in conf_lines:
if l.startswith("rpcuser="):
settings["username"] = l[8:].rstrip('\n')
if l.startswith("rpcpassword="):
settings["password"] = l[12:].rstrip('\n')
if l.startswith("rpcport="):
settings["rpc_port"] = int(l[8:].rstrip('\n'))
rpc_user = settings["username"]
rpc_pass = settings["password"]
rpc_port = settings["rpc_port"]
rpc_url = "127.0.0.1"
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
class LBRYumWallet(object):
def __init__(self, lbryum_path):
self.config = SimpleConfig()
self.config.set_key('chain', 'lbrycrd_main')
self.storage = WalletStorage(lbryum_path)
self.wallet = Wallet(self.storage)
self.cmd_runner = Commands(self.config, self.wallet, None)
if not self.wallet.has_seed():
seed = self.wallet.make_seed()
self.wallet.add_seed(seed, "derp")
self.wallet.create_master_keys("derp")
self.wallet.create_main_account()
self.wallet.update_password("derp", "")
self.network = Network(self.config)
self.blockchain = get_blockchain(self.config, self.network)
print self.config.get('chain'), self.blockchain
self.wallet.storage.write()
def command(self, command_name, *args, **kwargs):
cmd_runner = Commands(self.config, self.wallet, None)
cmd = known_commands[command_name]
func = getattr(cmd_runner, cmd.name)
return func(*args, **kwargs)
def generate_address(self):
address = self.wallet.create_new_address()
self.wallet.storage.write()
return address
class LBRYcrd(object):
def __init__(self, lbrycrdd_path):
self.lbrycrdd_conn_str = get_lbrycrdd_connection_string(lbrycrdd_path)
def __call__(self, method, *args, **kwargs):
return self.rpc(method)(*args, **kwargs)
def rpc(self, method):
return AuthServiceProxy(self.lbrycrdd_conn_str, service_name=method)
def get_wallet_path():
cwd = os.getcwd()
wallet_path = os.path.join(cwd, "wallet.json")
if not os.path.exists(wallet_path):
return wallet_path
i = 1
while True:
wallet_path = os.path.join(cwd, "wallet_%i.json" % i)
if not os.path.exists(wallet_path):
return wallet_path
i += 1
def coin_chooser(lbrycrdd, amount, fee=0.001):
def iter_txis():
unspent = lbrycrdd("listunspent")
unspent = sorted(unspent, key=lambda x: x['amount'], reverse=True)
spendable = Decimal(0.0)
for txi in unspent:
if spendable >= amount:
break
else:
spendable += txi['amount']
yield txi
if spendable < amount:
print spendable, amount
raise Exception("Not enough funds")
coins = list(iter(iter_txis()))
total = sum(c['amount'] for c in coins)
change = Decimal(total) - Decimal(amount) - Decimal(fee)
if change < 0:
raise Exception("Not enough funds")
if change:
change_address = lbrycrdd("getnewaddress")
else:
change_address = None
print "Total: %f, amount: %f, change: %f" % (total, amount, change)
return coins, change, change_address
def get_raw_tx(lbrycrdd, addresses, coins, amount, change, change_address):
txi = [{'txid': c['txid'], 'vout': c['vout']} for c in coins]
txo = {address: float(amount) for address in addresses}
if change_address:
txo[change_address] = float(change)
return lbrycrdd("createrawtransaction", txi, txo)
def main(count, value=None, lbryum_path=None, lbrycrdd_path=None):
count = int(count)
lbryum_path = lbryum_path or get_wallet_path()
if sys.platform == "darwin":
default_lbrycrdd = os.path.join(os.path.expanduser("~"),
"Library/Application Support/lbrycrd/lbrycrd.conf")
else:
default_lbrycrdd = os.path.join(os.path.expanduser("~"), ".lbrycrd/lbrycrd.conf")
lbrycrdd_path = lbrycrdd_path or default_lbrycrdd
l = LBRYcrd(lbrycrdd_path=lbrycrdd_path)
s = LBRYumWallet(lbryum_path)
value = value or 1.0
value = Decimal(value)
coins, change, change_address = coin_chooser(l, count * value)
addresses = [s.generate_address() for i in range(count)]
raw_tx = get_raw_tx(l, addresses, coins, value, change, change_address)
signed = l("signrawtransaction", raw_tx)['hex']
txid = l("sendrawtransaction", signed)
print txid
if __name__ == "__main__":
args = sys.argv[1:]
main(*args)
+408 -10
View File
@@ -140,19 +140,417 @@ func (ts *tagsSanitizer) add() {
}
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
anupjanakpur = "UC_5tRfC4L2AbTz6mj6vZrKw"
PraveenMedia = "UC_fjE70lKNwM9AKIofQv-bA"
MisteriosDesvelados = "UC-FzxivscjYzonBDXgX2GLg"
kaipenchery = "UC-MU4K3Ghl-IdEX4J68tnTA"
Dhinchakpooja = "UC-stzLwoQF_R8Vnfb28d7Lg"
Karolajn = "UC-vYDJQiabg9BK8XTDPFspg"
thefantasio974 = "UC0GHeUcPxfNEZqbFyxy2frQ"
EduPrimitivo = "UC0odmP6ffEw3iVwTEyt-UKA"
isupportanupriyapatel = "UC0sxOzdHnmauMdOra0cxXVg"
theconfusedindian = "UC12ZPYxQbMA1loZE1OA609w"
guriaarts = "UC1fHp166o1Hd024fg3-d-jg"
minutodafisica = "UC1lUEcHrQwbusC5ext1himg"
khabir = "UC1yzNCXXk1h_tHgsqChE8mw"
OMapadaMinaMara = "UC2WMUPTbxQQ9hXq2_uutspg"
oliverjanich = "UC3cmEfpy4XED7YYEe69nIMA"
shyamsadhu = "UC3XAT8oBjL2RaqfblESHW_A"
EbaduRahmantech = "UC4950gpY6Qw1lCAcN6gNWwQ"
_1975oles = "UC4bGQWN4C8idymxlZxw77oA"
_8dsongsbollywood = "UC4nH7zmw41lRDbaiIddumjw"
EYESTV = "UC50CIbyHMydEEzEhV1ZNEBw"
Nono = "UC5yufoRPJy-1e1pR73UTxFQ"
lichtle = "UC6n_2v7YjwZ65F9h77nZPOA"
elabuelokraken = "UC7iWk2xziMR4hs_NzfsYytw"
cidvela = "UCaVOx5GCcSi2ELjWUdf05PA"
jayaskumar = "UCax9CJCQ6aY0bntP3mcIWKQ"
canaldarippi = "UCB_fba7yYMwa91F7rijBsVA"
FunMusicClassical = "UCB_X256IN7QiBtDkueo_6QQ"
minutodaterra = "UCB0zinWfy-dS_NqcOINYo3A"
franciscoalves = "UCbAm6YcZGk04obnFb-LuOSA"
criptomonedastv = "UCbK6Awel1-o-9JDVMFq808Q"
lux77 = "UCBYm4l3NX352goFtjSgZ7jA"
MrLuckyOficial = "UCEbMhGhZ1JoOYKgbgVyeQaA"
KotneKit = "UCf_1BQz3T151Eph0Sarj3Yw"
jaysgaming = "UCFAkedtc3jjDZQqqIULxqWg"
iamdineshthakur = "UCg0VInmUZoSdHD-DNmjifBA"
Musicreationz = "UCGJEtZB0Nj3rT3zZfyfjIZQ"
lafenixtv = "UCHAx8o0jj-8L_sqR5QM1Cwg"
guardeiafe = "UCHWjmG-Ce93VNcNZMMUyGQQ"
KHANEKIKHUSHBOO = "UCI8DRKcrfrHklRqSJHPhOwQ"
AjayGamingYT = "UCi9r5igvQblbI0PhJFj-IDg"
SzekelyVegan = "UCiqvKH1Ib_VPwh9bQf_hlXQ"
barzoy888 = "UCIYF1orTg4nDvv29ORoKXyA"
_7playstudiopersian = "UCJ3QUKcd9yOhiJzZBR2zT3Q"
eazypurple = "UCJ6Hd8S39g7pe65A6k42NZw"
GennadyM = "UCjZlOrmC7hi_1kMA6BKZXeQ"
GTGujarati = "UCKnYZmYFdFX4hP09Hh8k8jw"
annapavlidou = "UCKPSHyCFIDtMqzbJ_Qx-7aw"
Tamilmemesclub = "UCL_DPW38jcSGWy0Fu0A_hCg"
QuickReactionTeam = "UCl--LVcANGKHJYN7O8uY6OA"
rodrigojoseoficial = "UClMsSb57Jvexl1a0a2c2_CQ"
AzadChaiwala = "UCM5QNdoIefx6eumjPk8ZTMw"
lafenixextras = "UCM7zougvyPtfd9zw5HmAGlg"
JustSawAGhost = "UCmQYXE03n6qlF9FhtAPztcw"
mirchibangla = "UCmzj6hXrPZ_AwIZ8lgo-HuQ"
marsaguirre = "UCN3OMvdU7ySvE6meGjo9omg"
JustGoool = "UCn7kTV_syU_rI0jANxrpaFg"
nossocanal = "UCNDb9jdx5Jz-C7hWJv3y6Fw"
CSNN = "UCnfn-8PJbjYFAJ1fqzMdcew"
pedronedved = "UCNhb2Fz3sVLZcMb_arm1cMQ"
famoosh = "UCnQU2QZLVG1wxHoy_9LZucA"
Cenoritas = "UCo-VSGoYy_4IjXF9u4QqTiw"
fantabobgames = "UCo0U1tbk3YbqiLDhkeWOviQ"
thesoundofsoul = "UCOey6R7Ktnil8RZrY_xJYiA"
SaurabhMishraJournalist = "UCoHVCBoSfUOlQVa80dlN60g"
YoYoHoneySinghlive = "UCOZTp3nGj39-snzo4QSdduA"
CINEGLITZ = "UCP_p8JkLOwPcGAnxN1iVhZA"
lovetreta = "UCPi3uuGrh2mxnZJMfRhcjiw"
canalvendimeusofa = "UCpk58NDdaKdX0QiiA2e79tg"
CARIBEANVIDEOS = "UCPynGkfzH35RKyXRccCgzNg"
Dimon = "UCpZsZZ6KEqCHeAJ_Y0gEbyQ"
Anantvijaysoni = "UCQ3NeFKF2yo70xxBBUDeXXQ"
canalilhadebarbados = "UCR5L_Q8Tiljy7WKEQOrGKbg"
SotomayorTV2 = "UCR70CjRHxQilfEUgBBYpDhg"
Recipesarab = "UCRbMDfkH_bPUjQsv5dHcFQA"
_4ak10 = "UCsbrIjDPPXuVlWApYTnQECA"
Top50 = "UCSjXR8uvU4PijMM_kgyVQfw"
SoumyajitPyne = "UCSJXwgF3SePNfyojYG3yt6Q"
SneksShow = "UCsqZSkVccnyIxSE6pZuj13g"
casadosaber = "UCtvvTFp0XANyllOdmzZr9VQ"
Entarexyt = "UCuDkq6yAb5zdM2zIRG6VRDw"
UCurU6GLM4ggcLtWWAOTzlYw = "UCurU6GLM4ggcLtWWAOTzlYw"
promining = "UCviqKxlMnZqBipTzwixkGyg"
zapkids = "UCx2uGhYa9EuCNmbL72hVKGg"
Canalokok = "UCXyCAcjoWz9SMLGJT8dR0pA"
SmoothGames = "UCY8gvG25rOZ4hWWvcQH88PQ"
LIGATFA = "UCYhiIMOlDn_HEn3zvqZCDiw"
DeHamad = "UCZ5G07Vw7IaV81InBgYLCcg"
misszizi = "UCE55WTFs4ekJ_aWCoNEapbQ"
creationshub = "UCNfELkowZPIQ-Vegmb3pIBQ"
AlicePandora = "UCScRxEtwlt082_6ThI0YJbg"
Akito0405 = "UCXHWx1teSYIKwTGiscYaC-Q"
dashcambristol = "UCD0IC8bZI-MfIutkgPHlyWA"
adrianbonillap = "UCdmY7p_kC-QN8jS7rocmCSA"
SubhamVlogs = "UCXtdoCLRlnLIW3Skix0rHaQ"
DisciplesofJesusChristJeremiahPayne = "UC_y2rVsotcQcVF7LImVGs5Q"
ModernGalaxy = "UCGISiGs_RL7Z1qfs1-GD6PA"
socofilms = "UCyDS9p6NWHpU9XbbbYLFLBw"
buriedone = "UC_lm7xXB3adOTc0T1FwyGRQ"
bitcoin = "UC-f5nPBEDyUBZz_jOPBwAfQ"
globalrashid = "UC2ldcEtbR7cFYadgrnW3B6Q"
gameofbitcoins = "UC2WKsYBxMwx7E7ENkND-fkg"
CryptoInsightsBrasil = "UC4BrnREinCBUenQZi4ZU95w"
Crypt0 = "UCdUSSt-IEUg2eq46rD7lu_g"
cryptomined = "UCGQ3XHtsH8Q9iQr9bFbgfDA"
altcoinbuzz = "UCGyqEtcGQQtXyUwvcy7Gmyg"
crypto = "UCiMgF08KQ4z-Gnu8o2BLOxA"
Crypto99 = "UCjsrdOJCAKcuBqyyqjg1cCQ"
TheCryptoLark = "UCl2oCaw8hdR_kbqyqd2klIA"
NuggetsNews = "UCLo66QVfEod0nNM_GzKNxmQ"
btckyle = "UCNCGCxxTT10aeTgUMHW5FfQ"
LouisThomas = "UCpceefaJ9vs4RYUTsO9Y3FA"
BitsBeTrippin = "UCVVWXoQfMfQVuzxcLylq9aA"
cryptocrow = "UCwsRWmIL5XKqFtdytBfeX0g"
KouSuccessLeeFX = "UC0YkP4Fg_d8y6JFwLj3MLdg"
TheSchiffReport = "UCIjuLiLHdFxYtFmWlbTGQRQ"
MRU = "UCnkEhPBMZcEO0QGu51fDFDg"
Vidello = "UCwMaWqZ6SdDpTaYOW0huELw"
kaccreative = "UC02O9ICMuwrfULSa0N6SiSw"
shecooks = "UChZYqTJkeYV2r7WETcytdPQ"
vegsource = "UClEsPxvotpTJ1Z8eu2Y97rg"
VeganGains = "UCr2eKhGzPhN5RPVk5dd5o3g"
AwesomeKnowledge = "UC_pC4T8vr-caiNDYU6PYTTA"
luckyloush = "UCbO0Oomf_jr20Wb4V2ap5Uw"
MichaelLuzzi = "UCDwLrj4DSGrw3gFvANbl_Cw"
NakedApe = "UCMOaRU-YsXVgU-WahBkZqWQ"
ParaReact = "UCuJKELjsmWTlJG0X7T4ZD_A"
dullytv = "UCv1J91Nhn7KsxMFaxKChT3w"
REDONKULAS = "UCwd_sSDZ8EQt6SEeOO2tBRA"
RedactedTonight = "UCyvaZ2RHEDrgKXz43gz7CbQ"
TonyTornado = "UCZu9AV3mrCCDpK_dy1qJRAQ"
hoodlumscrafty = "UC0cTVjYKgAnBrXQKcICyNmA"
Karmakut = "UC2B8TOklu2rSDULqAzwn5GQ"
Draxr = "UC72o3j23E1wKskBDEKlmTOw"
BigfryTV = "UC7FVdUA3SxDMfl4fLBGlADg"
CSGONews = "UC7L6NRLyldvxWukhOHABazQ"
ZedGaming = "UC7Q15H71DmB7F1iSFE1Z8LQ"
KamiVS = "UC9fh15yUcGAr8iUfQaoRRpQ"
grabthegames = "UCaJFEgY6ij05Fxgn6qtcX_g"
blueplays = "UCbjMsFlYb2NLpjS3uDzm9ow"
ImNotBonkers = "UCCyN0G77B7wnf-1AOTI3gWQ"
GunslingerMedia = "UCDjs4JoXmMzvaZyPVsvjKXw"
LeagueVoices = "UCdkonRjBLzr1Adf7Jhu1bWQ"
SirPugger = "UCelqWKTcCvBZP_1_1iYZAWA"
gamesoup = "UCGPMrF9AN_D9BrmSmMeV3hA"
ProHenis = "UCIjFoXSQ9HYbcWmRVIsH-Ow"
Dota2Divine = "UCiR9IHCurqVHpC821NVcW6g"
CrazyFoxMovies = "UCjewtQLpJEENPLbrCtb6YpA"
Larry = "UCJVdNvvuvOnthuWVQjYff2w"
hottake = "UCK24784Oqb4oYQfHL9XePNg"
Breezy = "UCKWRpZpcLKriWd1am9SHf8A"
retrorgb = "UCLPIbBCKVH2uKGm5C4sOkew"
Zer0Gamer = "UCmII34jN4rqCIsGqWkK5JEg"
nickatnyte = "UCMxYQX1zaepCgmiSmwbT39w"
DyllonStejGaming = "UCngaLL0QDbsAGYj7zsB8o_Q"
bluedrake42 = "UCNSwcDEUfIEzYdAPscXo6ZA"
Rerez = "UCoFpRCAsKfWAshvLE1bYzdw"
Op = "UCowi5kFfvGXR8NqhyE6jneQ"
kidsgamesfun = "UCqjGzmb2pMWSFYm0GzKeTEA"
GamesGlitches = "UCRj3Q06KOxAZWHaHScrkAOw"
Musikage = "UCsej4tgCoXDgVH3J7M3NMgw"
JeffyGaming = "UCTZzSNnZ43XQslejT5wFdRw"
oniblackmage = "UCUEF9XL3o8dZ6hvVf8jAi8Q"
nickatnyte2 = "UCuoTqrobMyZj0ge8LOCkiSw"
KazeEmanuar = "UCuvSqzfO_LV_QzHdmEj84SQ"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
GaminGHD = "UCW-thz5HxE-goYq8yPds1Gw"
BHGaming = "UCX4N3DioqqrugFeilxTkSIw"
Potato = "UCxPPTDNH85HZWxrgZ3FQBYA"
MikeyTaylorGaming = "UCycXj6lRWtsSqo-bZOIZePw"
Acituanbus = "UCzfx1QvKjn-BxLMLBdBGgMw"
juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
KhanAcademy = "UC4a-Gbdw7vOaccHmFo40b9g"
DON = "UCAYrPk70AePJZSaVLKrWdfQ"
shogogarcia = "UCE3yZjxDg3iI91bcNJDFnsg"
alphalifestyleacademy = "UCeggEaXtJu2domMahYMD_ig"
NileRed = "UCFhXFikryT4aFcLkLw2LBLA"
veritasium = "UCHnyfMqiRRG1u-2MsSQLbXA"
stevecronin = "UCJYawZQYwjZ76mrF_US9eNg"
jeranism = "UCS_FY5mR4g22L_E9t1D_ExQ"
MinutePhysics = "UCUHW94eEFW7hkUMVaZz4eDg"
_3Blue1Brown = "UCYO_jab_esuFRV4b17AJtAw"
Itsrucka = "UC-B2LyEZcl3avG0coKeohGQ"
unitedtaps = "UC5Q8e9-uutVZmiwAcEHKuzA"
srodalmenara = "UC9kc1DaOy2kSzXHZgpn9kfQ"
ShutupAndPlay = "UCAwuvzhah0KUw5QNihSkEwQ"
sanx2 = "UCJ_waKl9kjbhfXI3LJOfHvA"
DerickWattsAndTheSundayBlues = "UCmZhiZq7M7d73Kbey4yna_Q"
akirathedon = "UCsoiSpBvkr4Y-78Pj3recUw"
Musicoterapia = "UCsoSK8K4OpdMV1tqJFwO5QA"
daydreaming = "UCtbuGylbRXc42pIxWey19Dg"
RemixHolicRecords = "UCUW5GjwcXgbPcfk00t-GQZA"
EDMBot = "UCvmUdL2NHWlj1NRiNJPI-TQ"
TrioTravels = "UCdAPAmdnkdFsH5R2Hxevucg"
caosonnguyen = "UCwPeW9kFId5-VbQ2LQEjVhg"
TheAlmonteFilms = "UC4C_SF5koS4Q5om50b9NMTw"
timcast = "UCG749Dj4V2fKa143f8sE60Q"
SeekingTheTruth = "UCHrDpTVL9S0h91u9UCPgVbA"
davidpakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
mikenayna = "UCzk08fzh5c_BhjQa1w35wtA"
DoorMonster = "UC-to_wlckb-bFDtQfUZL3Kw"
barnacules = "UC1MwJy1R0nGQkXxRD9p-zTQ"
brightsunfilms = "UC5k3Kc0avyDJ2nG9Kxm9JmQ"
Onision = "UC5OxQNCgW88FDBxeZCnrBbg"
top10archive = "UCa03bf8gAS2EtffptV-_jfA"
thought = "UCb0yiUQhhLV_jpY3BayJaLA"
MothersBasement = "UCBs2Y3i14e1NWQxOGliatmg"
VlogsOfKnowledge = "UCDUPGR6uL5uz0_hiRAQjZMQ"
GorTheMovieGod = "UCHdTVw89QU6coU1MgN-9RHA"
iamalexoconnor = "UCKAQLEk1GGqnPtov9EW0huQ"
ADedits = "UCsX-zRuq3ovMsgFqEQLw2Bw"
SynthCool = "UCxGTHsD0pLSFlFI7M7jYmBQ"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
Sciencedocumentinhindi = "UC9SpfUF3rm-MGep5WE6FSCA"
NurdRage = "UCIgKGGJkt1MrNmhq3vRibYA"
MINDBLASTER = "UC_ZMqbRu44jK-EogjYyHz8g"
DaminousPurity = "UCdKRWvz50QoioZFgu6Nf9og"
ScammerRevolts = "UC0uJKUXiU5T41Fzawy5H6mw"
eevblog = "UC2DjFE7Xf11URZqWBigcVOQ"
Luke = "UC2eYFnH61tmytImy1mTYvhA"
AppGirl = "UC389S4_2Yt9cei1qNDwmBrA"
thecryptodad = "UC68x_TIzqCtF69fYl2_kl3w"
ChrisWereDigital = "UCAPR27YUyxmgwm3Wc2WSHLw"
alecaddd = "UCbmBY_XYZqCa2G0XmFA7ZWg"
EliTheComputerGuy = "UCD4EOyXKjfDUhCI6jlOZZYQ"
archetapp = "UCDIBBmkZIB2hjBsk1hUImdA"
PCPlaceNZ = "UCf5ZTSZAKbinY03jOwylfOg"
NaomiSexyCyborgWu = "UCh_ugKacslKhsGGdXP0cRRA"
NibiruWatcher = "UCi62JvN-lUn7hVL3ffofADA"
imineblocks = "UCjYHcWGAjUVqU49D2JOKD3w"
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
CooLoserTech = "UCl97rZ2Tc7KV9lktmmHNFDQ"
TechHD = "UCN3bPy04Jkp3ADRtyYvXomQ"
MiketheScrapper = "UCqtlJpkH_llXS_vuDExGVvw"
eevblog2 = "UCr-cm90DwFJC0W3f9jBs5jA"
thecreativeone = "UCTikFhzCiIXfOMS7D29dvYg"
weekendtricks = "UCYtAJXx0ymGPpCndn2Gt6-w"
GBGuns = "UC2VOURrALs1CwVmbGlXJOPQ"
Matsimus = "UCFWjEwhX6cSAKBQ28pufG3w"
TheLateBoyScout = "UCZjvj5MN3BMxPFfdEKIrvxQ"
BravoCinematografica = "UC2ruSXQoKMgr7JXzwO0H0KA"
PyroNation = "UC4ffy3n1hE7Z8q-2KVq_91Q"
dramatuber = "UC4Y8mImty3gFEG9grsR6T-w"
BarnabasNagy = "UC8TRZRK1sJfxKJ0tXMmGTow"
avery = "UCcfjIZLDCuSqkIlkH8i4DDg"
Tingledove = "UCfI7wtV6K64gVbzjH7DOA_Q"
crmjewelers = "UChpFWeF84jA5JeV3YyIo3pQ"
NerfNerd18 = "UCIgmaEJNqvH9bU9zGMapZGg"
SEIJIHITO = "UCNqUrLE6dI8fWw_u3HQkpXA"
anvithavlogs = "UCsP9pYat2DEBvnvF_iFGG_w"
YoelRekts = "UCZ_BcFyhIo6GdtTSrqXXepg"
TechFox = "UCIp-oTSdFO7BhAJpW2d5HMQ"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
Juggling: {"juggling", "circus arts", "malabares"},
JustJuggling: {"juggling", "circus arts", "malabares"},
JordanBPeterson: {"postmodernism", "psychology", "news"},
JustJuggling: {"juggling", "circus arts", "malabares"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
TechFox: {"technology", "reviews"},
misszizi: {"art", "pop culture"},
creationshub: {"art"},
AlicePandora: {"art"},
Akito0405: {"art"},
dashcambristol: {"automotive"},
adrianbonillap: {"automotive"},
SubhamVlogs: {"automotive"},
DisciplesofJesusChristJeremiahPayne: {"beliefs"},
ModernGalaxy: {"beliefs"},
socofilms: {"beliefs"},
buriedone: {"blockchain"},
bitcoin: {"blockchain"},
globalrashid: {"blockchain"},
gameofbitcoins: {"blockchain"},
CryptoInsightsBrasil: {"blockchain"},
Crypt0: {"blockchain"},
cryptomined: {"blockchain"},
altcoinbuzz: {"blockchain", "technology"},
crypto: {"blockchain"},
Crypto99: {"blockchain"},
TheCryptoLark: {"blockchain", "technology"},
NuggetsNews: {"blockchain", "learning"},
btckyle: {"blockchain"},
LouisThomas: {"blockchain"},
BitsBeTrippin: {"blockchain"},
cryptocrow: {"blockchain"},
KouSuccessLeeFX: {"economics", "learning"},
TheSchiffReport: {"economics"},
MRU: {"economics"},
Vidello: {"economics", "pop culture"},
kaccreative: {"food", "art"},
shecooks: {"food"},
vegsource: {"food"},
VeganGains: {"food"},
AwesomeKnowledge: {"funny"},
luckyloush: {"funny"},
MichaelLuzzi: {"funny", "pop culture"},
NakedApe: {"funny"},
ParaReact: {"funny"},
dullytv: {"funny"},
REDONKULAS: {"funny"},
RedactedTonight: {"funny", "news"},
TonyTornado: {"funny"},
hoodlumscrafty: {"gaming", "pop culture"},
Karmakut: {"gaming"},
Draxr: {"gaming", "pop culture"},
BigfryTV: {"gaming"},
CSGONews: {"gaming", "funny"},
ZedGaming: {"gaming"},
KamiVS: {"gaming"},
grabthegames: {"gaming"},
blueplays: {"gaming"},
ImNotBonkers: {"gaming", "funny"},
GunslingerMedia: {"gaming"},
LeagueVoices: {"gaming"},
SirPugger: {"gaming"},
gamesoup: {"gaming"},
ProHenis: {"gaming"},
Dota2Divine: {"gaming"},
CrazyFoxMovies: {"gaming", "pop culture", "technology"},
Larry: {"gaming"},
hottake: {"gaming", "funny"},
Breezy: {"gaming", "funny"},
retrorgb: {"gaming"},
Zer0Gamer: {"gaming"},
nickatnyte: {"gaming"},
DyllonStejGaming: {"gaming"},
bluedrake42: {"gaming"},
Rerez: {"gaming"},
Op: {"gaming"},
kidsgamesfun: {"gaming", "pop culture"},
GamesGlitches: {"gaming"},
Musikage: {"gaming", "pop culture"},
JeffyGaming: {"gaming", "funny"},
oniblackmage: {"gaming"},
nickatnyte2: {"gaming"},
KazeEmanuar: {"gaming"},
TheLinuxGamer: {"gaming", "technology", "linux"},
GaminGHD: {"gaming", "pop culture"},
BHGaming: {"gaming"},
Potato: {"gaming"},
MikeyTaylorGaming: {"gaming"},
Acituanbus: {"gaming"},
juggling: {"juggling", "circus art", "malabares"},
KhanAcademy: {"learning", "science"},
DON: {"learning", "pop culture"},
shogogarcia: {"learning"},
alphalifestyleacademy: {"learning"},
NileRed: {"learning", "science"},
veritasium: {"learning", "science"},
stevecronin: {"learning"},
jeranism: {"learning"},
MinutePhysics: {"learning", "science"},
_3Blue1Brown: {"learning"},
Itsrucka: {"music"},
unitedtaps: {"music"},
srodalmenara: {"music"},
ShutupAndPlay: {"music", "learning"},
sanx2: {"music"},
DerickWattsAndTheSundayBlues: {"music", "funny"},
akirathedon: {"music"},
Musicoterapia: {"music"},
daydreaming: {"music"},
RemixHolicRecords: {"music"},
EDMBot: {"music"},
TrioTravels: {"nature"},
caosonnguyen: {"nature"},
TheAlmonteFilms: {"news"},
timcast: {"news", "technology"},
SeekingTheTruth: {"news"},
davidpakman: {"news"},
mikenayna: {"news"},
DoorMonster: {"pop culture", "funny"},
barnacules: {"pop culture", "gaming"},
brightsunfilms: {"pop culture"},
Onision: {"pop culture", "funny"},
top10archive: {"pop culture"},
thought: {"pop culture", "learning"},
MothersBasement: {"pop culture", "gaming"},
VlogsOfKnowledge: {"pop culture", "gaming"},
GorTheMovieGod: {"pop culture"},
iamalexoconnor: {"pop culture"},
ADedits: {"pop culture"},
SynthCool: {"pop culture", "funny"},
JordanBPeterson: {"psychology", "postmodernism", "news"},
Sciencedocumentinhindi: {"science"},
NurdRage: {"science", "learning"},
MINDBLASTER: {"sports", "funny"},
DaminousPurity: {"sports", "gaming"},
ScammerRevolts: {"technology"},
eevblog: {"technology", "learning"},
Luke: {"technology", "funny"},
AppGirl: {"technology"},
thecryptodad: {"technology", "blockchain"},
ChrisWereDigital: {"technology"},
alecaddd: {"technology", "learning"},
EliTheComputerGuy: {"technology"},
archetapp: {"technology", "learning"},
PCPlaceNZ: {"technology"},
NaomiSexyCyborgWu: {"technology"},
NibiruWatcher: {"technology", "learning"},
imineblocks: {"technology", "blockchain"},
Lunduke: {"technology"},
CooLoserTech: {"technology"},
TechHD: {"technology", "learning"},
MiketheScrapper: {"technology"},
eevblog2: {"technology"},
thecreativeone: {"technology", "gaming"},
weekendtricks: {"technology"},
GBGuns: {"weapons"},
Matsimus: {"weapons", "gaming"},
TheLateBoyScout: {"weapons"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,