Compare commits

...
9 Commits
Author SHA1 Message Date
Niko Storni 95890cbabf prevent live streams from syncing
reduce timeout to 20 minutes
2020-04-29 16:15:22 +02:00
Niko Storni 69c2a91ec7 fix bug in video length 2020-04-23 16:37:35 +02:00
Niko Storni fbe09a692c fix 403 error 2020-04-21 20:56:14 +02:00
Niko Storni 0ccc6e7979 add temporary logging 2020-04-21 16:26:34 +02:00
Niko Storni 3eb74cc8b5 log video length failures 2020-04-19 02:38:31 +02:00
Niko Storni 9bfb1656fe fix length checks on single syncs 2020-04-10 17:46:07 +02:00
Niko Storni 5d40d24804 fix max length issue for bigger youtubers
fix issue with no-playlist-found channels
reduce the max db size limit
2020-04-08 23:14:10 +02:00
Niko Storni 79d3aadda8 add utxo_release as hotfix for stuck balances 2020-04-01 04:49:24 +02:00
Niko Storni eba59a0806 change utxo_list signature 2020-04-01 04:44:05 +02:00
8 changed files with 32 additions and 10 deletions

No files matched your search

+1 -1
View File
@@ -20,7 +20,7 @@ require (
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/lbryio/lbry.go/v2 v2.5.1
github.com/lbryio/lbry.go/v2 v2.5.3
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
+2 -2
View File
@@ -188,8 +188,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.5.1 h1:Rb3BbvXKhtpTsXizCGqtCBHt/e5vDUuMNkzALfql03U=
github.com/lbryio/lbry.go/v2 v2.5.1/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
github.com/lbryio/lbry.go/v2 v2.5.3 h1:Qw4APDuVZDkwjQ49usy4p/ty/KfuQr18ld6GiREYlog=
github.com/lbryio/lbry.go/v2 v2.5.3/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
+4 -1
View File
@@ -135,6 +135,7 @@ func (s *SyncManager) Start() error {
ConcurrentVideos: s.concurrentVideos,
Refill: s.refill,
Manager: s,
MaxVideoLength: s.maxVideoLength,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
@@ -168,8 +169,9 @@ func (s *SyncManager) Start() error {
}
for i, c := range channels {
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
maxVideoLength := s.maxVideoLength
if c.TotalSubscribers < 1000 {
s.maxVideoLength = 1.0
maxVideoLength = 1.0
}
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
@@ -178,6 +180,7 @@ func (s *SyncManager) Start() error {
lbryChannelID: c.ChannelClaimID,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
MaxVideoLength: maxVideoLength,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
+1 -1
View File
@@ -196,7 +196,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
for _, utxo := range utxolist.Items {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
confirmedCount++
}
+1 -1
View File
@@ -86,7 +86,7 @@ func abandonSupports(s *Sync) (float64, error) {
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
defer s.daemon.SetRPCTimeout(40 * time.Minute)
defer s.daemon.SetRPCTimeout(20 * time.Minute)
for i := 0; i < s.ConcurrentVideos; i++ {
consumerWG.Add(1)
go func() {
+15 -3
View File
@@ -91,6 +91,7 @@ type Sync struct {
clientPublishAddress string
publicKey string
defaultAccountID string
MaxVideoLength float64
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -241,7 +242,7 @@ func (s *Sync) setStatusSyncing() error {
func (s *Sync) setExceptions() {
if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
s.Manager.maxVideoLength = 0.0 // skips max length checks
s.MaxVideoLength = 9999.0 // skips max length checks
s.Manager.maxVideoSize = 0
}
}
@@ -306,7 +307,7 @@ func (s *Sync) FullCycle() (e error) {
log.Infoln("Waiting for daemon to finish starting...")
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
s.daemon.SetRPCTimeout(40 * time.Minute)
s.daemon.SetRPCTimeout(20 * time.Minute)
err = s.waitForDaemonStart()
if err != nil {
@@ -781,6 +782,10 @@ func (s *Sync) doSync() error {
if err != nil {
return errors.Prefix("could not import the transferee public key", err)
}
_, err = s.daemon.UTXORelease(nil)
if err != nil {
return errors.Prefix("could not run uxo_release", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
@@ -899,6 +904,7 @@ func (s *Sync) startWorker(workerNum int) {
"interrupted by user",
"Sign in to confirm your age",
"This video is unavailable",
"video is a live stream and hasn't completed yet",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -975,6 +981,8 @@ func (s *Sync) startWorker(workerNum int) {
}
}
var mostRecentlyFailedChannel string
func (s *Sync) enqueueYoutubeVideos() error {
client := &http.Client{
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
@@ -1029,6 +1037,10 @@ func (s *Sync) enqueueYoutubeVideos() error {
if youtubeIsLying {
break
}
if s.YoutubeChannelID == mostRecentlyFailedChannel {
return errors.Err("playlist items not found")
}
mostRecentlyFailedChannel = s.YoutubeChannelID
break //return errors.Err("playlist items not found") //TODO: will this work?
}
videoIDs := make([]string, 50)
@@ -1154,7 +1166,7 @@ func (s *Sync) processVideo(v video) (err error) {
ChannelID: s.lbryChannelID,
MaxVideoSize: s.Manager.maxVideoSize,
Namer: s.namer,
MaxVideoLength: s.Manager.maxVideoLength,
MaxVideoLength: s.MaxVideoLength,
Fee: s.Fee,
DefaultAccount: da,
}
+7
View File
@@ -17,6 +17,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
@@ -179,6 +180,9 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
}
func (v *YoutubeVideo) download() error {
if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
return errors.Err("video is a live stream and hasn't completed yet")
}
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777)
@@ -204,6 +208,7 @@ func (v *YoutubeVideo) download() error {
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
"--rm-cache-dir",
"--postprocessor-args",
"-movflags faststart",
"--abort-on-unavailable-fragment",
@@ -439,6 +444,8 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
return nil, errors.Err(err)
}
if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
return nil, errors.Err("video is too long to process")
}
for {
+1 -1
View File
@@ -253,7 +253,7 @@ func CleanupLbrynet() error {
}
return errors.Err(err)
}
dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
if db.Size() > dbSizeLimit {
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
if err != nil {