Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
14668c339e | ||
|
|
7d38aa7b29 | ||
|
|
3c3ea2138e |
No files matched your search
@@ -84,7 +84,7 @@ services:
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:transfers
|
||||
image: lbry/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
|
||||
+3
-5
@@ -70,11 +70,11 @@ mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
|
||||
# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "0" ]]; then
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
echo "Video Status: $videoStatus"
|
||||
@@ -88,6 +88,4 @@ if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStat
|
||||
exit 1;
|
||||
else
|
||||
echo "SUCCESSSSSSSSSSSSS!"
|
||||
fi;
|
||||
|
||||
#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address
|
||||
fi;
|
||||
@@ -21,24 +21,18 @@ var Version string
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
singleRun bool
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
flags sdk.SyncFlags
|
||||
maxTries int
|
||||
refill int
|
||||
limit int
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -52,15 +46,16 @@ func main() {
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
@@ -100,7 +95,7 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
return
|
||||
}
|
||||
|
||||
if stopOnError && maxTries != defaultMaxTries {
|
||||
if flags.StopOnError && maxTries != defaultMaxTries {
|
||||
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
|
||||
return
|
||||
}
|
||||
@@ -168,13 +163,10 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
HostName: hostname,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
stopOnError,
|
||||
flags,
|
||||
maxTries,
|
||||
takeOverExistingChannel,
|
||||
refill,
|
||||
limit,
|
||||
skipSpaceCheck,
|
||||
syncUpdate,
|
||||
concurrentJobs,
|
||||
concurrentJobs,
|
||||
blobsDir,
|
||||
@@ -186,12 +178,9 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
awsS3Region,
|
||||
awsS3Bucket,
|
||||
syncStatus,
|
||||
singleRun,
|
||||
syncProperties,
|
||||
apiConfig,
|
||||
maxVideoLength,
|
||||
removeDBUnpublished,
|
||||
upgradeMetadata,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
|
||||
+75
-92
@@ -20,61 +20,48 @@ import (
|
||||
)
|
||||
|
||||
type SyncManager struct {
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
singleRun bool
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
SyncFlags sdk.SyncFlags
|
||||
maxTries int
|
||||
refill int
|
||||
limit int
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
}
|
||||
|
||||
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
|
||||
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
|
||||
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
|
||||
syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
|
||||
return &SyncManager{
|
||||
stopOnError: stopOnError,
|
||||
maxTries: maxTries,
|
||||
takeOverExistingChannel: takeOverExistingChannel,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
skipSpaceCheck: skipSpaceCheck,
|
||||
syncUpdate: syncUpdate,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
singleRun: singleRun,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
removeDBUnpublished: removeDBUnpublished,
|
||||
upgradeMetadata: upgradeMetadata,
|
||||
SyncFlags: syncFlags,
|
||||
maxTries: maxTries,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,25 +126,23 @@ func (s *SyncManager) Start() error {
|
||||
lbryChannelName := channels[0].DesiredChannelName
|
||||
syncs = make([]Sync, 1)
|
||||
syncs[0] = Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
publishAddress: channels[0].PublishAddress,
|
||||
transferState: channels[0].TransferState,
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
publishAddress: channels[0].PublishAddress,
|
||||
transferState: channels[0].TransferState,
|
||||
}
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
@@ -165,7 +150,7 @@ func (s *SyncManager) Start() error {
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.syncUpdate {
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
|
||||
} else {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
|
||||
@@ -183,25 +168,23 @@ func (s *SyncManager) Start() error {
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
publishAddress: c.PublishAddress,
|
||||
transferState: c.TransferState,
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
publishAddress: c.PublishAddress,
|
||||
transferState: c.TransferState,
|
||||
})
|
||||
if q != StatusFailed {
|
||||
continue queues
|
||||
@@ -248,7 +231,7 @@ func (s *SyncManager) Start() error {
|
||||
break
|
||||
}
|
||||
}
|
||||
if shouldInterruptLoop || s.singleRun {
|
||||
if shouldInterruptLoop || s.SyncFlags.SingleRun {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -265,7 +248,7 @@ func (s *SyncManager) checkUsedSpace() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if usedPctile >= 0.90 && !s.skipSpaceCheck {
|
||||
if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
|
||||
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
|
||||
}
|
||||
log.Infof("disk usage: %.1f%%", usedPctile*100)
|
||||
|
||||
+1
-1
@@ -95,7 +95,7 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
|
||||
if s.Manager.upgradeMetadata {
|
||||
if s.Manager.SyncFlags.UpgradeMetadata {
|
||||
requiredBalance += float64(notUpgradedCount) * 0.001
|
||||
}
|
||||
|
||||
|
||||
+15
-5
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
@@ -33,18 +34,19 @@ waiting:
|
||||
return nil
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) error {
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
supports, err := s.daemon.SupportList(nil, page, 50)
|
||||
if err != nil {
|
||||
return errors.Prefix("cannot list claims", err)
|
||||
return 0, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
}
|
||||
alreadyAbandoned := make(map[string]bool, len(allSupports))
|
||||
totalAbandoned := 0.0
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
@@ -53,11 +55,19 @@ func abandonSupports(s *Sync) error {
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
summary, err := s.daemon.SupportAbandon(&support.ClaimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
return totalAbandoned, errors.Err(err)
|
||||
}
|
||||
log.Infof("Abandoned support of %s (%s total output) LBC for claim %s", support.Amount, summary.Outputs[0].Amount, support.ClaimID)
|
||||
if len(summary.Outputs) < 1 {
|
||||
return totalAbandoned, errors.Err("error abandoning supports: no outputs while abandoning %s", support.ClaimID)
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
return totalAbandoned, errors.Err(err)
|
||||
}
|
||||
totalAbandoned += outputAmount
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, support.ClaimID)
|
||||
}
|
||||
return nil
|
||||
return totalAbandoned, nil
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
|
||||
+56
-36
@@ -63,33 +63,31 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
|
||||
|
||||
// Sync stores the options that control how syncing happens
|
||||
type Sync struct {
|
||||
APIConfig *sdk.APIConfig
|
||||
YoutubeChannelID string
|
||||
LbryChannelName string
|
||||
StopOnError bool
|
||||
MaxTries int
|
||||
ConcurrentVideos int
|
||||
TakeOverExistingChannel bool
|
||||
Refill int
|
||||
Manager *SyncManager
|
||||
LbrycrdString string
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
transferState int
|
||||
publishAddress string
|
||||
APIConfig *sdk.APIConfig
|
||||
YoutubeChannelID string
|
||||
LbryChannelName string
|
||||
MaxTries int
|
||||
ConcurrentVideos int
|
||||
Refill int
|
||||
Manager *SyncManager
|
||||
LbrycrdString string
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
transferState int
|
||||
publishAddress string
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -320,15 +318,35 @@ func (s *Sync) FullCycle() (e error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = abandonSupports(s)
|
||||
supportAmount, err := abandonSupports(s)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
|
||||
}
|
||||
err = transferVideos(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return transferChannel(s)
|
||||
err = transferChannel(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reallocateSupports := supportAmount > 0.01
|
||||
if reallocateSupports {
|
||||
err = waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -344,7 +362,7 @@ func deleteSyncFolder(videoDirectory string) {
|
||||
}
|
||||
}
|
||||
func (s *Sync) shouldTransfer() bool {
|
||||
return s.transferState == 1 && s.publishAddress != ""
|
||||
return s.transferState == 1 && s.publishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
|
||||
}
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
var transferState *int
|
||||
@@ -556,11 +574,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
||||
}
|
||||
_, ok := videoIDMap[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
}
|
||||
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
|
||||
if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
|
||||
log.Infof("removing: %s", strings.Join(idsToRemove, ","))
|
||||
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
|
||||
if err != nil {
|
||||
return count, fixed, len(idsToRemove), err
|
||||
@@ -666,7 +685,7 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
}
|
||||
|
||||
if s.StopOnError {
|
||||
if s.Manager.SyncFlags.StopOnError {
|
||||
log.Println("Will stop publishing if an error is detected")
|
||||
}
|
||||
|
||||
@@ -732,7 +751,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError {
|
||||
s.grp.Stop()
|
||||
} else if s.MaxTries > 1 {
|
||||
errorsNoRetry := []string{
|
||||
@@ -754,6 +773,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"download error: ERROR: Sorry about that",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -957,7 +977,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
s.syncedVideosMux.RUnlock()
|
||||
newMetadataVersion := int8(2)
|
||||
alreadyPublished := ok && sv.Published
|
||||
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
|
||||
neverRetryFailures := []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
|
||||
+11
@@ -34,6 +34,17 @@ type SyncProperties struct {
|
||||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
StopOnError bool
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
SingleRun bool
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
|
||||
@@ -519,8 +519,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
videoSize = uint64(existingVideoData.Size)
|
||||
} else {
|
||||
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
|
||||
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
if !v.mocked {
|
||||
_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
|
||||
Reference in new issue
Block a user