Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65daeec216 | ||
|
|
ad1704fb39 | ||
|
|
44e6cb5ddc | ||
|
|
5171acc007 | ||
|
|
946314da94 | ||
|
|
5961327adf | ||
|
|
d2bbb36312 | ||
|
|
c7272e988a | ||
|
|
5b9846bcca | ||
|
|
8d849b8f06 | ||
|
|
c41c368f34 | ||
|
|
7ce4c4a2ce | ||
|
|
0d473cc4f6 | ||
|
|
2f701a887a | ||
|
|
8d56ef852f | ||
|
|
849ff11bfd | ||
|
|
45982f30c5 | ||
|
|
07413b499a | ||
|
|
92d8ef75f8 | ||
|
|
1b08bb0e61 | ||
|
|
bdd55c9965 | ||
|
|
14668c339e | ||
|
|
7d38aa7b29 | ||
|
|
3c3ea2138e | ||
|
|
7b23235f83 |
No files matched your search
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.39.3
|
||||
image: lbry/wallet-server:v0.42.0
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.39.3
|
||||
image: lbry/lbrynet:v0.42.0
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
@@ -84,7 +84,7 @@ services:
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:transfers
|
||||
image: lbry/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
|
||||
+4
-6
@@ -67,14 +67,14 @@ curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params
|
||||
# Reset status for tranfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc'
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
|
||||
# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "0" ]]; then
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
echo "Video Status: $videoStatus"
|
||||
@@ -88,6 +88,4 @@ if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStat
|
||||
exit 1;
|
||||
else
|
||||
echo "SUCCESSSSSSSSSSSSS!"
|
||||
fi;
|
||||
|
||||
#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address
|
||||
fi;
|
||||
@@ -12,7 +12,7 @@ require (
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/go-ini/ini v1.41.0 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303
|
||||
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
|
||||
@@ -159,6 +159,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/Xo
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 h1:CyDDxUMREhAxPlgP+mgcArgkGJKtdXssj7CXk7o3h84=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85 h1:jBcKTwo0J10GzKBriDEUnT/we1edaaFZWtB7o57D9jI=
|
||||
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85/go.mod h1:wnoiUIfkeUmbiIGR/dg3Xvb6a6Jz7vPxtUoENyyl5QE=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
|
||||
|
||||
@@ -21,24 +21,18 @@ var Version string
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
singleRun bool
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
flags sdk.SyncFlags
|
||||
maxTries int
|
||||
refill int
|
||||
limit int
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -52,15 +46,16 @@ func main() {
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
@@ -100,7 +95,7 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
return
|
||||
}
|
||||
|
||||
if stopOnError && maxTries != defaultMaxTries {
|
||||
if flags.StopOnError && maxTries != defaultMaxTries {
|
||||
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
|
||||
return
|
||||
}
|
||||
@@ -168,13 +163,10 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
HostName: hostname,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
stopOnError,
|
||||
flags,
|
||||
maxTries,
|
||||
takeOverExistingChannel,
|
||||
refill,
|
||||
limit,
|
||||
skipSpaceCheck,
|
||||
syncUpdate,
|
||||
concurrentJobs,
|
||||
concurrentJobs,
|
||||
blobsDir,
|
||||
@@ -186,12 +178,9 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
awsS3Region,
|
||||
awsS3Bucket,
|
||||
syncStatus,
|
||||
singleRun,
|
||||
syncProperties,
|
||||
apiConfig,
|
||||
maxVideoLength,
|
||||
removeDBUnpublished,
|
||||
upgradeMetadata,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
|
||||
+78
-92
@@ -20,61 +20,48 @@ import (
|
||||
)
|
||||
|
||||
type SyncManager struct {
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
singleRun bool
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
SyncFlags sdk.SyncFlags
|
||||
maxTries int
|
||||
refill int
|
||||
limit int
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
}
|
||||
|
||||
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
|
||||
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
|
||||
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
|
||||
syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
|
||||
return &SyncManager{
|
||||
stopOnError: stopOnError,
|
||||
maxTries: maxTries,
|
||||
takeOverExistingChannel: takeOverExistingChannel,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
skipSpaceCheck: skipSpaceCheck,
|
||||
syncUpdate: syncUpdate,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
singleRun: singleRun,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
removeDBUnpublished: removeDBUnpublished,
|
||||
upgradeMetadata: upgradeMetadata,
|
||||
SyncFlags: syncFlags,
|
||||
maxTries: maxTries,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,25 +126,24 @@ func (s *SyncManager) Start() error {
|
||||
lbryChannelName := channels[0].DesiredChannelName
|
||||
syncs = make([]Sync, 1)
|
||||
syncs[0] = Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
publishAddress: channels[0].PublishAddress,
|
||||
transferState: channels[0].TransferState,
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
clientPublishAddress: channels[0].PublishAddress,
|
||||
publicKey: channels[0].PublicKey,
|
||||
transferState: channels[0].TransferState,
|
||||
}
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
@@ -165,7 +151,7 @@ func (s *SyncManager) Start() error {
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.syncUpdate {
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
|
||||
} else {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
|
||||
@@ -183,25 +169,24 @@ func (s *SyncManager) Start() error {
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
publishAddress: c.PublishAddress,
|
||||
transferState: c.TransferState,
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
clientPublishAddress: c.PublishAddress,
|
||||
publicKey: c.PublicKey,
|
||||
transferState: c.TransferState,
|
||||
})
|
||||
if q != StatusFailed {
|
||||
continue queues
|
||||
@@ -226,6 +211,7 @@ func (s *SyncManager) Start() error {
|
||||
"failure uploading wallet",
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
@@ -248,7 +234,7 @@ func (s *SyncManager) Start() error {
|
||||
break
|
||||
}
|
||||
}
|
||||
if shouldInterruptLoop || s.singleRun {
|
||||
if shouldInterruptLoop || s.SyncFlags.SingleRun {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -265,7 +251,7 @@ func (s *SyncManager) checkUsedSpace() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if usedPctile >= 0.90 && !s.skipSpaceCheck {
|
||||
if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
|
||||
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
|
||||
}
|
||||
log.Infof("disk usage: %.1f%%", usedPctile*100)
|
||||
|
||||
+24
-15
@@ -94,14 +94,19 @@ func (s *Sync) walletSetup() error {
|
||||
videosOnYoutube = s.Manager.videosLimit
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
|
||||
if s.Manager.upgradeMetadata {
|
||||
channelFee := channelClaimAmount
|
||||
channelAlreadyClaimed := s.lbryChannelID != ""
|
||||
if channelAlreadyClaimed {
|
||||
channelFee = 0.0
|
||||
}
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
|
||||
if s.Manager.SyncFlags.UpgradeMetadata {
|
||||
requiredBalance += float64(notUpgradedCount) * 0.001
|
||||
}
|
||||
|
||||
refillAmount := 0.0
|
||||
if balance < requiredBalance || balance < minimumAccountBalance {
|
||||
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
|
||||
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
|
||||
}
|
||||
|
||||
if s.Refill > 0 {
|
||||
@@ -115,18 +120,18 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
}
|
||||
|
||||
claimAddress, err := s.daemon.AddressList(nil)
|
||||
claimAddress, err := s.daemon.AddressList(nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get unused address")
|
||||
}
|
||||
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
|
||||
s.claimAddress = string((*claimAddress)[0].Address)
|
||||
if s.claimAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
if s.transferState > 0 && s.publishAddress != "" {
|
||||
s.claimAddress = s.publishAddress
|
||||
if s.shouldTransfer() {
|
||||
s.claimAddress = s.clientPublishAddress
|
||||
}
|
||||
|
||||
err = s.ensureEnoughUTXOs()
|
||||
@@ -136,11 +141,10 @@ func (s *Sync) walletSetup() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
accounts, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
accountsNet := (*accounts).LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
@@ -154,7 +158,15 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
}
|
||||
if defaultAccount == "" {
|
||||
return errors.Err("No default account found")
|
||||
return "", errors.Err("No default account found")
|
||||
}
|
||||
return defaultAccount, nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
@@ -265,10 +277,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if s.LbryChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
//@TODO: get rid of this when imported channels are supported
|
||||
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.transferState == TransferStateComplete {
|
||||
return nil
|
||||
}
|
||||
|
||||
+64
-18
@@ -1,18 +1,24 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allConfirmed := false
|
||||
waiting:
|
||||
for !allConfirmed {
|
||||
utxolist, err := s.daemon.UTXOList(nil)
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
@@ -33,38 +39,55 @@ waiting:
|
||||
return nil
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) error {
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
supports, err := s.daemon.SupportList(nil, page, 50)
|
||||
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return errors.Prefix("cannot list claims", err)
|
||||
return 0, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
}
|
||||
alreadyAbandoned := make(map[string]bool, len(allSupports))
|
||||
totalAbandoned := 0.0
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
supportOnTransferredClaim := support.Address == s.clientPublishAddress //todo: probably not needed anymore
|
||||
if supportOnTransferredClaim {
|
||||
continue
|
||||
}
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
summary, err := s.daemon.SupportAbandon(&support.ClaimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
return totalAbandoned, errors.Err(err)
|
||||
}
|
||||
log.Infof("Abandoned support of %s (%s total output) LBC for claim %s", support.Amount, summary.Outputs[0].Amount, support.ClaimID)
|
||||
if len(summary.Outputs) < 1 {
|
||||
return totalAbandoned, errors.Err("error abandoning supports: no outputs while abandoning %s", support.ClaimID)
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
return totalAbandoned, errors.Err(err)
|
||||
}
|
||||
totalAbandoned += outputAmount
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, support.ClaimID)
|
||||
}
|
||||
return nil
|
||||
return totalAbandoned, nil
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
cleanTransfer := true
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
|
||||
log.Debugf("skipping video: %s", video.VideoID)
|
||||
//log.Debugf("skipping video: %s", video.VideoID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -79,11 +102,14 @@ func transferVideos(s *Sync) error {
|
||||
return errors.Err("cannot transfer: too many claims. claimID: %s", video.ClaimID)
|
||||
}
|
||||
|
||||
if c.Claims[0].Address == s.clientPublishAddress {
|
||||
continue
|
||||
}
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.publishAddress},
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
},
|
||||
Bid: util.PtrToString("0.009"), // Todo - Dont hardcode
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
|
||||
videoStatus := sdk.VideoStatus{
|
||||
@@ -101,14 +127,16 @@ func transferVideos(s *Sync) error {
|
||||
videoStatus.FailureReason = updateError.Error()
|
||||
videoStatus.Status = VideoStatusTranferFailed
|
||||
videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
|
||||
log.Infof("TRANSFERRED %t", *videoStatus.IsTransferred)
|
||||
statusErr := s.APIConfig.MarkVideoStatus(videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Err(statusErr)
|
||||
return errors.Prefix(statusErr.Error(), updateError)
|
||||
}
|
||||
if updateError != nil {
|
||||
return errors.Err(err)
|
||||
return errors.Err(updateError)
|
||||
}
|
||||
}
|
||||
// Todo - Transfer Channel as last step and post back to remote db that channel is transferred.
|
||||
@@ -118,23 +146,41 @@ func transferVideos(s *Sync) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func transferChannel(s *Sync) error {
|
||||
channelClaim, err := s.daemon.ClaimSearch(nil, &s.lbryChannelID, nil, nil)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
channelClaims, err := s.daemon.ChannelList(&account, 1, 50)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if channelClaim == nil || len(channelClaim.Claims) == 0 {
|
||||
return errors.Err("There is no channel claim for channel %s", s.LbryChannelName)
|
||||
var channelClaim *jsonrpc.Transaction = nil
|
||||
for _, c := range channelClaims.Items {
|
||||
if c.ClaimID != s.lbryChannelID {
|
||||
continue
|
||||
}
|
||||
channelClaim = &c
|
||||
break
|
||||
}
|
||||
if channelClaim == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
updateOptions := jsonrpc.ChannelUpdateOptions{
|
||||
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.publishAddress,
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
},
|
||||
},
|
||||
}
|
||||
result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
|
||||
|
||||
return errors.Err(err)
|
||||
return nil
|
||||
}
|
||||
+270
-112
@@ -8,6 +8,7 @@ import (
|
||||
"os/signal"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
@@ -30,7 +31,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/mitchellh/go-ps"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
@@ -39,7 +39,7 @@ import (
|
||||
const (
|
||||
channelClaimAmount = 0.01
|
||||
estimatedMaxTxFee = 0.1
|
||||
minimumAccountBalance = 4.0
|
||||
minimumAccountBalance = 1.0
|
||||
minimumRefillAmount = 1
|
||||
publishAmount = 0.01
|
||||
maxReasonLength = 500
|
||||
@@ -63,33 +63,32 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
|
||||
|
||||
// Sync stores the options that control how syncing happens
|
||||
type Sync struct {
|
||||
APIConfig *sdk.APIConfig
|
||||
YoutubeChannelID string
|
||||
LbryChannelName string
|
||||
StopOnError bool
|
||||
MaxTries int
|
||||
ConcurrentVideos int
|
||||
TakeOverExistingChannel bool
|
||||
Refill int
|
||||
Manager *SyncManager
|
||||
LbrycrdString string
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
transferState int
|
||||
publishAddress string
|
||||
APIConfig *sdk.APIConfig
|
||||
YoutubeChannelID string
|
||||
LbryChannelName string
|
||||
MaxTries int
|
||||
ConcurrentVideos int
|
||||
Refill int
|
||||
Manager *SyncManager
|
||||
LbrycrdString string
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
transferState int
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -316,24 +315,53 @@ func (s *Sync) FullCycle() (e error) {
|
||||
}
|
||||
|
||||
if s.shouldTransfer() {
|
||||
err := waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = abandonSupports(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = transferVideos(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return transferChannel(s)
|
||||
return s.processTransfers()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) processTransfers() (e error) {
|
||||
log.Println("Processing transfers")
|
||||
err := waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
supportAmount, err := abandonSupports(s)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
|
||||
}
|
||||
if supportAmount > 0 {
|
||||
logUtils.SendInfoToSlack("%.6f LBCs were abandoned and should be used as support", supportAmount)
|
||||
}
|
||||
err = transferVideos(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = transferChannel(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reallocateSupports := supportAmount > 0.01
|
||||
if reallocateSupports {
|
||||
err = waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
}
|
||||
}
|
||||
log.Println("Done processing transfers")
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteSyncFolder(videoDirectory string) {
|
||||
if !strings.Contains(videoDirectory, "/tmp/ytsync") {
|
||||
_ = util.SendToSlack(errors.Err("Trying to delete an unexpected directory: %s", videoDirectory).Error())
|
||||
@@ -343,9 +371,11 @@ func deleteSyncFolder(videoDirectory string) {
|
||||
_ = util.SendToSlack(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Sync) shouldTransfer() bool {
|
||||
return s.transferState == 1 && s.publishAddress != ""
|
||||
return s.transferState >= 1 && s.clientPublishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
|
||||
}
|
||||
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
var transferState *int
|
||||
|
||||
@@ -475,10 +505,14 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
claimToAbandon = cl
|
||||
videoIDs[videoID] = c
|
||||
}
|
||||
log.Debugf("abandoning %+v", claimToAbandon)
|
||||
_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
|
||||
if err != nil {
|
||||
return true, err
|
||||
if claimToAbandon.Address != s.clientPublishAddress && !s.syncedVideos[videoID].Transferred {
|
||||
log.Debugf("abandoning %+v", claimToAbandon)
|
||||
_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
} else {
|
||||
log.Debugf("lbrynet stream abandon --txid=%s --nout=%d", claimToAbandon.Txid, claimToAbandon.Nout)
|
||||
}
|
||||
abandonedClaims = true
|
||||
//return true, nil
|
||||
@@ -486,49 +520,84 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
return abandonedClaims, nil
|
||||
}
|
||||
|
||||
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
|
||||
//additionally it removes all entries in the database indicating that a video is published when it's actually not
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
|
||||
count := 0
|
||||
videoIDMap := make(map[string]string, len(claims))
|
||||
type ytsyncClaim struct {
|
||||
ClaimID string
|
||||
MetadataVersion uint
|
||||
ClaimName string
|
||||
PublishAddress string
|
||||
VideoID string
|
||||
Claim *jsonrpc.Claim
|
||||
}
|
||||
|
||||
// mapFromClaims returns a map of videoIDs (youtube id) to ytsyncClaim which is a structure holding blockchain related
|
||||
// information
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
//check if claimID is in remote db
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
videoIDMap[videoID] = c.ClaimID
|
||||
s.syncedVideosMux.RLock()
|
||||
pv, claimInDatabase := s.syncedVideos[videoID]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
claimMetadataVersion := uint(1)
|
||||
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
|
||||
claimMetadataVersion = 2
|
||||
}
|
||||
|
||||
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
|
||||
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
|
||||
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
|
||||
claimMarkedUnpublished := claimInDatabase && !pv.Published
|
||||
videoIDMap[videoID] = ytsyncClaim{
|
||||
ClaimID: c.ClaimID,
|
||||
MetadataVersion: claimMetadataVersion,
|
||||
ClaimName: c.Name,
|
||||
PublishAddress: c.Address,
|
||||
VideoID: videoID,
|
||||
Claim: &c,
|
||||
}
|
||||
}
|
||||
return videoIDMap
|
||||
}
|
||||
|
||||
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
|
||||
//additionally it removes all entries in the database indicating that a video is published when it's actually not
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim) (total, fixed, removed int, err error) {
|
||||
allClaimsInfo := s.mapFromClaims(claims)
|
||||
ownClaimsInfo := s.mapFromClaims(ownClaims)
|
||||
count := len(allClaimsInfo)
|
||||
idsToRemove := make([]string, 0, count)
|
||||
|
||||
for videoID, chainInfo := range allClaimsInfo {
|
||||
s.syncedVideosMux.RLock()
|
||||
sv, claimInDatabase := s.syncedVideos[videoID]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
|
||||
metadataDiffers := claimInDatabase && sv.MetadataVersion != int8(chainInfo.MetadataVersion)
|
||||
claimIDDiffers := claimInDatabase && sv.ClaimID != chainInfo.ClaimID
|
||||
claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
|
||||
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
||||
_, isOwnClaim := ownClaimsInfo[videoID]
|
||||
tranferred := !isOwnClaim
|
||||
transferStatusMismatch := sv.Transferred != tranferred
|
||||
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
||||
}
|
||||
if claimIDDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
|
||||
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, sv.ClaimID, chainInfo.ClaimID)
|
||||
}
|
||||
if claimNameDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
|
||||
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, sv.ClaimName, chainInfo.ClaimName)
|
||||
}
|
||||
if claimMarkedUnpublished {
|
||||
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
|
||||
}
|
||||
if !claimInDatabase {
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
|
||||
}
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
|
||||
claimSize, err := c.GetStreamSizeByMagic()
|
||||
if transferStatusMismatch {
|
||||
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
}
|
||||
@@ -538,42 +607,72 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
||||
ChannelID: s.YoutubeChannelID,
|
||||
VideoID: videoID,
|
||||
Status: VideoStatusPublished,
|
||||
ClaimID: c.ClaimID,
|
||||
ClaimName: c.Name,
|
||||
ClaimID: chainInfo.ClaimID,
|
||||
ClaimName: chainInfo.ClaimName,
|
||||
Size: util.PtrToInt64(int64(claimSize)),
|
||||
MetaDataVersion: claimMetadataVersion,
|
||||
MetaDataVersion: chainInfo.MetadataVersion,
|
||||
IsTransferred: &tranferred,
|
||||
})
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
idsToRemove := make([]string, 0, len(videoIDMap))
|
||||
|
||||
//reload the synced videos map before we use it for further processing
|
||||
if fixed > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
for vID, sv := range s.syncedVideos {
|
||||
if sv.Transferred {
|
||||
log.Infof("%s: claim was transferred, ignoring")
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
_, ok := videoIDMap[vID]
|
||||
_, ok := ownClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
}
|
||||
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
|
||||
if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
|
||||
log.Infof("removing: %s", strings.Join(idsToRemove, ","))
|
||||
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
|
||||
if err != nil {
|
||||
return count, fixed, len(idsToRemove), err
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return count, fixed, 0, nil
|
||||
//reload the synced videos map before we use it for further processing
|
||||
if removed > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return count, fixed, removed, err
|
||||
}
|
||||
}
|
||||
return count, fixed, removed, nil
|
||||
}
|
||||
|
||||
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
totalPages := uint64(1)
|
||||
var allClaims []jsonrpc.Claim
|
||||
var account *string = nil
|
||||
if defaultOnly {
|
||||
a, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
account = &a
|
||||
}
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
claims, err := s.daemon.ClaimList(nil, page, 50)
|
||||
claims, err := s.daemon.ClaimList(account, page, 50)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
@@ -584,7 +683,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
allClaims, err := s.getClaims()
|
||||
allClaims, err := s.getClaims(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -598,22 +697,22 @@ func (s *Sync) checkIntegrity() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allClaims, err = s.getClaims()
|
||||
allClaims, err = s.getClaims(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
pubsOnWallet, nFixed, nRemoved, err := s.updateRemoteDB(allClaims)
|
||||
ownClaims, err := s.getClaims(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pubsOnWallet, nFixed, nRemoved, err := s.updateRemoteDB(allClaims, ownClaims)
|
||||
if err != nil {
|
||||
return errors.Prefix("error updating remote database", err)
|
||||
}
|
||||
|
||||
if nFixed > 0 || nRemoved > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if nFixed > 0 {
|
||||
logUtils.SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
|
||||
}
|
||||
@@ -635,6 +734,11 @@ func (s *Sync) checkIntegrity() error {
|
||||
if pubsOnWallet < pubsOnDB {
|
||||
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
_, err = s.getUnsentSupports() //TODO: use the returned value when it works
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -643,6 +747,10 @@ func (s *Sync) doSync() error {
|
||||
if err != nil {
|
||||
return errors.Prefix("could not set address reuse policy", err)
|
||||
}
|
||||
err = s.importPublicKey()
|
||||
if err != nil {
|
||||
return errors.Prefix("could not import the transferee public key", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
@@ -666,7 +774,7 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
}
|
||||
|
||||
if s.StopOnError {
|
||||
if s.Manager.SyncFlags.StopOnError {
|
||||
log.Println("Will stop publishing if an error is detected")
|
||||
}
|
||||
|
||||
@@ -731,8 +839,9 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError {
|
||||
s.grp.Stop()
|
||||
} else if s.MaxTries > 1 {
|
||||
errorsNoRetry := []string{
|
||||
@@ -745,7 +854,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers)",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"no compatible format available for this video",
|
||||
@@ -754,6 +863,8 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -957,7 +1068,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
s.syncedVideosMux.RUnlock()
|
||||
newMetadataVersion := int8(2)
|
||||
alreadyPublished := ok && sv.Published
|
||||
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
|
||||
neverRetryFailures := []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
@@ -1025,39 +1136,86 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
|
||||
func waitForDaemonProcess(timeout time.Duration) error {
|
||||
processes, err := ps.Processes()
|
||||
if err != nil {
|
||||
return err
|
||||
func (s *Sync) importPublicKey() error {
|
||||
if s.publicKey != "" {
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
}
|
||||
for _, a := range accounts {
|
||||
if a.PublicKey == s.publicKey {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
|
||||
_, err = s.daemon.AccountAdd(s.LbryChannelName, nil, nil, &s.publicKey, util.PtrToBool(true), nil)
|
||||
return errors.Err(err)
|
||||
}
|
||||
var daemonProcessId = -1
|
||||
for _, p := range processes {
|
||||
if p.Executable() == "lbrynet" {
|
||||
daemonProcessId = p.Pid()
|
||||
break
|
||||
return nil
|
||||
}
|
||||
|
||||
//TODO: fully implement this once I find a way to reliably get the abandoned supports amount
|
||||
func (s *Sync) getUnsentSupports() (float64, error) {
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if s.transferState == 2 {
|
||||
balance, err := s.daemon.AccountBalance(&defaultAccount)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
} else if balance == nil {
|
||||
return 0, errors.Err("no response")
|
||||
}
|
||||
|
||||
balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
transactionList, err := s.daemon.TransactionList(&defaultAccount)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
sentSupports := 0.0
|
||||
for _, t := range *transactionList {
|
||||
if len(t.SupportInfo) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, support := range t.SupportInfo {
|
||||
supportAmount, err := strconv.ParseFloat(support.BalanceDelta, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if supportAmount < 0 { // && support.IsTip TODO: re-enable this when transaction list shows correct information
|
||||
sentSupports += -supportAmount
|
||||
}
|
||||
}
|
||||
}
|
||||
if balanceAmount > 10 && sentSupports < 1 {
|
||||
logUtils.SendErrorToSlack("this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", balanceAmount, sentSupports)
|
||||
return balanceAmount - 10, nil
|
||||
}
|
||||
}
|
||||
if daemonProcessId == -1 {
|
||||
return nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
|
||||
func waitForDaemonProcess(timeout time.Duration) error {
|
||||
then := time.Now()
|
||||
stopTime := then.Add(time.Duration(timeout * time.Second))
|
||||
for !time.Now().After(stopTime) {
|
||||
wait := 10 * time.Second
|
||||
log.Println("the daemon is still running, waiting for it to exit")
|
||||
time.Sleep(wait)
|
||||
proc, err := os.FindProcess(daemonProcessId)
|
||||
running, err := logUtils.IsLbrynetRunning()
|
||||
if err != nil {
|
||||
// couldn't find the process, that means the daemon is stopped and can continue
|
||||
return nil
|
||||
return errors.Err(err)
|
||||
}
|
||||
//double check if process is running and alive
|
||||
//by sending a signal 0
|
||||
//NOTE : syscall.Signal is not available in Windows
|
||||
err = proc.Signal(syscall.Signal(0))
|
||||
//the process doesn't exist anymore! we're free to go
|
||||
if err != nil && (err == syscall.ESRCH || err.Error() == "os: process already finished") {
|
||||
if !running {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
+8
-1
@@ -43,9 +43,16 @@ func (n *Namer) GetNextName(prefix string) string {
|
||||
}
|
||||
|
||||
//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
|
||||
attempt = 1
|
||||
if len(name) < 2 {
|
||||
sum := md5.Sum([]byte(prefix))
|
||||
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
|
||||
for {
|
||||
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
|
||||
if _, exists := n.names[name]; !exists {
|
||||
break
|
||||
}
|
||||
attempt++
|
||||
}
|
||||
}
|
||||
|
||||
n.names[name] = true
|
||||
|
||||
+12
@@ -34,6 +34,17 @@ type SyncProperties struct {
|
||||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
StopOnError bool
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
SingleRun bool
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
@@ -47,6 +58,7 @@ type YoutubeChannel struct {
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
TransferState int `json:"transfer_state"`
|
||||
PublishAddress string `json:"publish_address"`
|
||||
PublicKey string `json:"public_key"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
|
||||
|
||||
@@ -519,8 +519,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
videoSize = uint64(existingVideoData.Size)
|
||||
} else {
|
||||
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
|
||||
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
if !v.mocked {
|
||||
_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
|
||||
Reference in new issue
Block a user