Compare commits

...
21 Commits
Author SHA1 Message Date
Niko Storni 946314da94 update lbry.go to fix claim_search 2019-10-03 22:51:21 +02:00
Niko Storni 5961327adf reduce bid even further to avoid lockups 2019-10-03 21:06:47 +02:00
Niko Storni d2bbb36312 improve detection for own claims vs their claims 2019-10-01 16:55:43 +02:00
Niko Storni c7272e988a reduce amount to avoid dust 2019-09-27 20:17:49 +02:00
Niko Storni 5b9846bcca fix pointer issue 2019-09-27 19:29:42 +02:00
Niko Storni 8d849b8f06 do not retry unavailable videos 2019-09-27 16:12:21 +02:00
Niko Storni c41c368f34 fix transfer bugs 2019-09-27 16:10:57 +02:00
Niko Storni 7ce4c4a2ce fix lbrynet shutdown check in e2e 2019-09-25 05:07:19 +02:00
Niko Storni 0d473cc4f6 add support for transferring videos after transferring everything 2019-09-25 04:38:49 +02:00
Niko Storni 2f701a887a fix double counting
fix transferred check
2019-09-25 04:15:16 +02:00
Niko Storni 8d56ef852f fix e2e
fix database transferred flag
2019-09-25 03:30:07 +02:00
Niko Storni 849ff11bfd import public keys if available 2019-09-24 20:42:17 +02:00
Niko Storni 45982f30c5 fix e2e test 2019-09-24 19:09:39 +02:00
Niko Storni 07413b499a update SDK support
fix bug where videos would publish to the wrong address
2019-09-24 18:51:06 +02:00
Niko Storni 92d8ef75f8 update e2e test 2019-09-19 20:03:55 +02:00
Niko Storni 1b08bb0e61 fix bug with non latin titles for claim names
hard exit when a duplicate name is attempted
2019-09-10 11:43:20 +02:00
Niko Storni bdd55c9965 fix balance refill issue
add additional debugging
2019-09-04 19:24:11 +02:00
Niko Storni 14668c339e refactor sync flags
add disable-transfer flag
2019-08-30 21:31:58 +02:00
Niko Storni 7d38aa7b29 add republishing on non-available videos
enhanced logging
2019-08-30 19:35:04 +02:00
Niko Storni 3c3ea2138e adjust support management
adjust e2e script
2019-08-28 21:08:25 +02:00
Niko Storni 7b23235f83 Merge branch 'transfers' 2019-08-28 15:26:38 +02:00
12 changed files with 305 additions and 225 deletions

No files matched your search

+3 -3
View File
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.39.3
image: lbry/wallet-server:v0.42.0
restart: always
environment:
- DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.39.3
image: lbry/lbrynet:v0.42.0
restart: always
ports:
- "15100:5279"
@@ -84,7 +84,7 @@ services:
## Internal APIs ##
###################
internalapis:
image: lbry/internal-apis:transfers
image: lbry/internal-apis:master
restart: "no"
ports:
- "15400:8080"
+4 -6
View File
@@ -67,14 +67,14 @@ curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params
# Reset status for tranfer test
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
# Trigger transfer api
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc'
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
# Execute the transfer test!
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "0" ]]; then
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
echo "Video Status: $videoStatus"
@@ -88,6 +88,4 @@ if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStat
exit 1;
else
echo "SUCCESSSSSSSSSSSSS!"
fi;
#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address
fi;
+1 -1
View File
@@ -12,7 +12,7 @@ require (
github.com/docker/go-units v0.4.0 // indirect
github.com/go-ini/ini v1.41.0 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
+2
View File
@@ -159,6 +159,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/Xo
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 h1:CyDDxUMREhAxPlgP+mgcArgkGJKtdXssj7CXk7o3h84=
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85 h1:jBcKTwo0J10GzKBriDEUnT/we1edaaFZWtB7o57D9jI=
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85/go.mod h1:wnoiUIfkeUmbiIGR/dg3Xvb6a6Jz7vPxtUoENyyl5QE=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
+22 -33
View File
@@ -21,24 +21,18 @@ var Version string
const defaultMaxTries = 3
var (
stopOnError bool
maxTries int
takeOverExistingChannel bool
refill int
limit int
skipSpaceCheck bool
syncUpdate bool
singleRun bool
syncStatus string
channelID string
syncFrom int64
syncUntil int64
concurrentJobs int
videosLimit int
maxVideoSize int
maxVideoLength float64
removeDBUnpublished bool
upgradeMetadata bool
flags sdk.SyncFlags
maxTries int
refill int
limit int
syncStatus string
channelID string
syncFrom int64
syncUntil int64
concurrentJobs int
videosLimit int
maxVideoSize int
maxVideoLength float64
)
func main() {
@@ -52,15 +46,16 @@ func main() {
Args: cobra.RangeArgs(0, 0),
}
cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
@@ -100,7 +95,7 @@ func ytSync(cmd *cobra.Command, args []string) {
return
}
if stopOnError && maxTries != defaultMaxTries {
if flags.StopOnError && maxTries != defaultMaxTries {
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
return
}
@@ -168,13 +163,10 @@ func ytSync(cmd *cobra.Command, args []string) {
HostName: hostname,
}
sm := manager.NewSyncManager(
stopOnError,
flags,
maxTries,
takeOverExistingChannel,
refill,
limit,
skipSpaceCheck,
syncUpdate,
concurrentJobs,
concurrentJobs,
blobsDir,
@@ -186,12 +178,9 @@ func ytSync(cmd *cobra.Command, args []string) {
awsS3Region,
awsS3Bucket,
syncStatus,
singleRun,
syncProperties,
apiConfig,
maxVideoLength,
removeDBUnpublished,
upgradeMetadata,
)
err := sm.Start()
if err != nil {
+78 -92
View File
@@ -20,61 +20,48 @@ import (
)
type SyncManager struct {
stopOnError bool
maxTries int
takeOverExistingChannel bool
refill int
limit int
skipSpaceCheck bool
syncUpdate bool
concurrentJobs int
concurrentVideos int
blobsDir string
videosLimit int
maxVideoSize int
maxVideoLength float64
lbrycrdString string
awsS3ID string
awsS3Secret string
awsS3Region string
syncStatus string
awsS3Bucket string
singleRun bool
syncProperties *sdk.SyncProperties
apiConfig *sdk.APIConfig
removeDBUnpublished bool
upgradeMetadata bool
SyncFlags sdk.SyncFlags
maxTries int
refill int
limit int
concurrentJobs int
concurrentVideos int
blobsDir string
videosLimit int
maxVideoSize int
maxVideoLength float64
lbrycrdString string
awsS3ID string
awsS3Secret string
awsS3Region string
syncStatus string
awsS3Bucket string
syncProperties *sdk.SyncProperties
apiConfig *sdk.APIConfig
}
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
return &SyncManager{
stopOnError: stopOnError,
maxTries: maxTries,
takeOverExistingChannel: takeOverExistingChannel,
refill: refill,
limit: limit,
skipSpaceCheck: skipSpaceCheck,
syncUpdate: syncUpdate,
concurrentJobs: concurrentJobs,
concurrentVideos: concurrentVideos,
blobsDir: blobsDir,
videosLimit: videosLimit,
maxVideoSize: maxVideoSize,
maxVideoLength: maxVideoLength,
lbrycrdString: lbrycrdString,
awsS3ID: awsS3ID,
awsS3Secret: awsS3Secret,
awsS3Region: awsS3Region,
awsS3Bucket: awsS3Bucket,
syncStatus: syncStatus,
singleRun: singleRun,
syncProperties: syncProperties,
apiConfig: apiConfig,
removeDBUnpublished: removeDBUnpublished,
upgradeMetadata: upgradeMetadata,
SyncFlags: syncFlags,
maxTries: maxTries,
refill: refill,
limit: limit,
concurrentJobs: concurrentJobs,
concurrentVideos: concurrentVideos,
blobsDir: blobsDir,
videosLimit: videosLimit,
maxVideoSize: maxVideoSize,
maxVideoLength: maxVideoLength,
lbrycrdString: lbrycrdString,
awsS3ID: awsS3ID,
awsS3Secret: awsS3Secret,
awsS3Region: awsS3Region,
awsS3Bucket: awsS3Bucket,
syncStatus: syncStatus,
syncProperties: syncProperties,
apiConfig: apiConfig,
}
}
@@ -139,25 +126,24 @@ func (s *SyncManager) Start() error {
lbryChannelName := channels[0].DesiredChannelName
syncs = make([]Sync, 1)
syncs[0] = Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
LbryChannelName: lbryChannelName,
lbryChannelID: channels[0].ChannelClaimID,
StopOnError: s.stopOnError,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
TakeOverExistingChannel: s.takeOverExistingChannel,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: channels[0].Fee,
publishAddress: channels[0].PublishAddress,
transferState: channels[0].TransferState,
APIConfig: s.apiConfig,
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
LbryChannelName: lbryChannelName,
lbryChannelID: channels[0].ChannelClaimID,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: channels[0].Fee,
publishAddress: channels[0].PublishAddress,
publicKey: channels[0].PublicKey,
transferState: channels[0].TransferState,
}
shouldInterruptLoop = true
} else {
@@ -165,7 +151,7 @@ func (s *SyncManager) Start() error {
//TODO: implement scrambling to avoid starvation of queues
if s.syncStatus != "" {
queuesToSync = append(queuesToSync, s.syncStatus)
} else if s.syncUpdate {
} else if s.SyncFlags.SyncUpdate {
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
} else {
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
@@ -183,25 +169,24 @@ func (s *SyncManager) Start() error {
for i, c := range channels {
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: c.ChannelId,
LbryChannelName: c.DesiredChannelName,
lbryChannelID: c.ChannelClaimID,
StopOnError: s.stopOnError,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
TakeOverExistingChannel: s.takeOverExistingChannel,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: c.Fee,
publishAddress: c.PublishAddress,
transferState: c.TransferState,
APIConfig: s.apiConfig,
YoutubeChannelID: c.ChannelId,
LbryChannelName: c.DesiredChannelName,
lbryChannelID: c.ChannelClaimID,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
AwsS3Region: s.awsS3Region,
AwsS3Bucket: s.awsS3Bucket,
namer: namer.NewNamer(),
Fee: c.Fee,
publishAddress: c.PublishAddress,
publicKey: c.PublicKey,
transferState: c.TransferState,
})
if q != StatusFailed {
continue queues
@@ -226,6 +211,7 @@ func (s *SyncManager) Start() error {
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
@@ -248,7 +234,7 @@ func (s *SyncManager) Start() error {
break
}
}
if shouldInterruptLoop || s.singleRun {
if shouldInterruptLoop || s.SyncFlags.SingleRun {
break
}
}
@@ -265,7 +251,7 @@ func (s *SyncManager) checkUsedSpace() error {
if err != nil {
return errors.Err(err)
}
if usedPctile >= 0.90 && !s.skipSpaceCheck {
if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
}
log.Infof("disk usage: %.1f%%", usedPctile*100)
+22 -10
View File
@@ -94,14 +94,19 @@ func (s *Sync) walletSetup() error {
videosOnYoutube = s.Manager.videosLimit
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
if s.Manager.upgradeMetadata {
channelFee := channelClaimAmount
channelAlreadyClaimed := s.lbryChannelID != ""
if channelAlreadyClaimed {
channelFee = 0.0
}
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
if s.Manager.SyncFlags.UpgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
}
refillAmount := 0.0
if balance < requiredBalance || balance < minimumAccountBalance {
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
}
if s.Refill > 0 {
@@ -115,17 +120,17 @@ func (s *Sync) walletSetup() error {
}
}
claimAddress, err := s.daemon.AddressList(nil)
claimAddress, err := s.daemon.AddressList(nil, nil)
if err != nil {
return err
} else if claimAddress == nil {
return errors.Err("could not get unused address")
}
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
s.claimAddress = string((*claimAddress)[0].Address)
if s.claimAddress == "" {
return errors.Err("found blank claim address")
}
if s.transferState > 0 && s.publishAddress != "" {
if s.shouldTransfer() {
s.claimAddress = s.publishAddress
}
@@ -136,11 +141,10 @@ func (s *Sync) walletSetup() error {
return nil
}
func (s *Sync) ensureEnoughUTXOs() error {
func (s *Sync) getDefaultAccount() (string, error) {
accounts, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
return "", errors.Err(err)
}
accountsNet := (*accounts).LBCMainnet
if logUtils.IsRegTest() {
@@ -154,7 +158,15 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
}
if defaultAccount == "" {
return errors.Err("No default account found")
return "", errors.Err("No default account found")
}
return defaultAccount, nil
}
func (s *Sync) ensureEnoughUTXOs() error {
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
utxolist, err := s.daemon.UTXOList(&defaultAccount)
+39 -10
View File
@@ -1,11 +1,13 @@
package manager
import (
"fmt"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/sdk"
log "github.com/sirupsen/logrus"
"strconv"
)
func waitConfirmations(s *Sync) error {
@@ -33,31 +35,48 @@ waiting:
return nil
}
func abandonSupports(s *Sync) error {
func abandonSupports(s *Sync) (float64, error) {
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return 0, err
}
for page := uint64(1); page <= totalPages; page++ {
supports, err := s.daemon.SupportList(nil, page, 50)
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
if err != nil {
return errors.Prefix("cannot list claims", err)
return 0, errors.Prefix("cannot list claims", err)
}
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
}
alreadyAbandoned := make(map[string]bool, len(allSupports))
totalAbandoned := 0.0
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
continue
}
supportOnTransferredClaim := support.Address == s.publishAddress //todo: probably not needed anymore
if supportOnTransferredClaim {
continue
}
alreadyAbandoned[support.ClaimID] = true
summary, err := s.daemon.SupportAbandon(&support.ClaimID, nil, nil, nil, nil)
if err != nil {
return errors.Err(err)
return totalAbandoned, errors.Err(err)
}
log.Infof("Abandoned support of %s (%s total output) LBC for claim %s", support.Amount, summary.Outputs[0].Amount, support.ClaimID)
if len(summary.Outputs) < 1 {
return totalAbandoned, errors.Err("error abandoning supports: no outputs while abandoning %s", support.ClaimID)
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
return totalAbandoned, errors.Err(err)
}
totalAbandoned += outputAmount
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, support.ClaimID)
}
return nil
return totalAbandoned, nil
}
func transferVideos(s *Sync) error {
@@ -83,7 +102,7 @@ func transferVideos(s *Sync) error {
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.publishAddress},
},
Bid: util.PtrToString("0.009"), // Todo - Dont hardcode
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
videoStatus := sdk.VideoStatus{
@@ -101,14 +120,16 @@ func transferVideos(s *Sync) error {
videoStatus.FailureReason = updateError.Error()
videoStatus.Status = VideoStatusTranferFailed
videoStatus.IsTransferred = util.PtrToBool(false)
} else {
videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
}
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
log.Infof("TRANSFERRED %t", *videoStatus.IsTransferred)
statusErr := s.APIConfig.MarkVideoStatus(videoStatus)
if statusErr != nil {
return errors.Err(statusErr)
return errors.Prefix(statusErr.Error(), updateError)
}
if updateError != nil {
return errors.Err(err)
return errors.Err(updateError)
}
}
// Todo - Transfer Channel as last step and post back to remote db that channel is transferred.
@@ -118,6 +139,7 @@ func transferVideos(s *Sync) error {
}
return nil
}
func transferChannel(s *Sync) error {
channelClaim, err := s.daemon.ClaimSearch(nil, &s.lbryChannelID, nil, nil)
if err != nil {
@@ -126,7 +148,11 @@ func transferChannel(s *Sync) error {
if channelClaim == nil || len(channelClaim.Claims) == 0 {
return errors.Err("There is no channel claim for channel %s", s.LbryChannelName)
}
if channelClaim.Claims[0].Address == s.publishAddress {
return nil
}
updateOptions := jsonrpc.ChannelUpdateOptions{
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
ClaimAddress: &s.publishAddress,
@@ -134,6 +160,9 @@ func transferChannel(s *Sync) error {
},
}
result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions)
if err != nil {
return errors.Err(err)
}
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
return errors.Err(err)
+105 -67
View File
@@ -30,7 +30,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/mitchellh/go-ps"
log "github.com/sirupsen/logrus"
"google.golang.org/api/googleapi/transport"
"google.golang.org/api/youtube/v3"
@@ -39,7 +38,7 @@ import (
const (
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.1
minimumAccountBalance = 4.0
minimumAccountBalance = 1.0
minimumRefillAmount = 1
publishAmount = 0.01
maxReasonLength = 500
@@ -63,33 +62,32 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
// Sync stores the options that control how syncing happens
type Sync struct {
APIConfig *sdk.APIConfig
YoutubeChannelID string
LbryChannelName string
StopOnError bool
MaxTries int
ConcurrentVideos int
TakeOverExistingChannel bool
Refill int
Manager *SyncManager
LbrycrdString string
AwsS3ID string
AwsS3Secret string
AwsS3Region string
AwsS3Bucket string
Fee *sdk.Fee
daemon *jsonrpc.Client
claimAddress string
videoDirectory string
syncedVideosMux *sync.RWMutex
syncedVideos map[string]sdk.SyncedVideo
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.RWMutex
queue chan video
transferState int
publishAddress string
APIConfig *sdk.APIConfig
YoutubeChannelID string
LbryChannelName string
MaxTries int
ConcurrentVideos int
Refill int
Manager *SyncManager
LbrycrdString string
AwsS3ID string
AwsS3Secret string
AwsS3Region string
AwsS3Bucket string
Fee *sdk.Fee
daemon *jsonrpc.Client
claimAddress string
videoDirectory string
syncedVideosMux *sync.RWMutex
syncedVideos map[string]sdk.SyncedVideo
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.RWMutex
queue chan video
transferState int
publishAddress string
publicKey string
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -320,15 +318,35 @@ func (s *Sync) FullCycle() (e error) {
if err != nil {
return err
}
err = abandonSupports(s)
supportAmount, err := abandonSupports(s)
if err != nil {
return err
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
}
err = transferVideos(s)
if err != nil {
return err
}
return transferChannel(s)
err = transferChannel(s)
if err != nil {
return err
}
reallocateSupports := supportAmount > 0.01
if reallocateSupports {
err = waitConfirmations(s)
if err != nil {
return err
}
isTip := true
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
if err != nil {
return errors.Err(err)
}
if len(summary.Outputs) < 1 {
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
}
}
return nil
}
return nil
@@ -343,9 +361,11 @@ func deleteSyncFolder(videoDirectory string) {
_ = util.SendToSlack(err.Error())
}
}
func (s *Sync) shouldTransfer() bool {
return s.transferState == 1 && s.publishAddress != ""
return s.transferState >= 1 && s.publishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
}
func (s *Sync) setChannelTerminationStatus(e *error) {
var transferState *int
@@ -475,10 +495,14 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
claimToAbandon = cl
videoIDs[videoID] = c
}
log.Debugf("abandoning %+v", claimToAbandon)
_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
if err != nil {
return true, err
if claimToAbandon.Address != s.publishAddress && !s.syncedVideos[videoID].Transferred {
log.Debugf("abandoning %+v", claimToAbandon)
_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
if err != nil {
return true, err
}
} else {
log.Debugf("lbrynet stream abandon --txid=%s --nout=%d", claimToAbandon.Txid, claimToAbandon.Nout)
}
abandonedClaims = true
//return true, nil
@@ -527,6 +551,9 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
if !claimInDatabase {
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
}
if s.syncedVideos[videoID].Transferred && s.publishAddress != c.Address {
log.Debugf("%s: Marked as transferred while in fact it's not (%s - %s). Publish address: %s, expected: %s", videoID, c.Name, c.ClaimID, c.Address, s.publishAddress)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
claimSize, err := c.GetStreamSizeByMagic()
if err != nil {
@@ -542,6 +569,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
ClaimName: c.Name,
Size: util.PtrToInt64(int64(claimSize)),
MetaDataVersion: claimMetadataVersion,
IsTransferred: util.PtrToBool(s.publishAddress == c.Address),
})
if err != nil {
return count, fixed, 0, err
@@ -551,16 +579,17 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
idsToRemove := make([]string, 0, len(videoIDMap))
for vID, sv := range s.syncedVideos {
if sv.Transferred {
log.Infof("%s: claim was transferred, ignoring")
log.Infof("%s: claim was transferred, ignoring", vID)
continue
}
_, ok := videoIDMap[vID]
if !ok && sv.Published {
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
idsToRemove = append(idsToRemove, vID)
}
}
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
log.Infof("removing: %s", strings.Join(idsToRemove, ","))
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
if err != nil {
return count, fixed, len(idsToRemove), err
@@ -643,6 +672,10 @@ func (s *Sync) doSync() error {
if err != nil {
return errors.Prefix("could not set address reuse policy", err)
}
err = s.importPublicKey()
if err != nil {
return errors.Prefix("could not import the transferee public key", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
@@ -666,7 +699,7 @@ func (s *Sync) doSync() error {
}
}
if s.StopOnError {
if s.Manager.SyncFlags.StopOnError {
log.Println("Will stop publishing if an error is detected")
}
@@ -731,8 +764,9 @@ func (s *Sync) startWorker(workerNum int) {
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"Couldn't find private key for id",
"You already have a stream claim published under the name",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError {
s.grp.Stop()
} else if s.MaxTries > 1 {
errorsNoRetry := []string{
@@ -745,7 +779,7 @@ func (s *Sync) startWorker(workerNum int) {
"Error in daemon: Cannot publish empty file",
"Error extracting sts from embedded url response",
"Unable to extract signature tokens",
"Client.Timeout exceeded while awaiting headers)",
"Client.Timeout exceeded while awaiting headers",
"the video is too big to sync, skipping for now",
"video is too long to process",
"no compatible format available for this video",
@@ -754,6 +788,8 @@ func (s *Sync) startWorker(workerNum int) {
"the video must be republished as we can't get the right size",
"HTTP Error 403",
"giving up after 0 fragment retries",
"Sorry about that",
"This video is not available",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -957,7 +993,7 @@ func (s *Sync) processVideo(v video) (err error) {
s.syncedVideosMux.RUnlock()
newMetadataVersion := int8(2)
alreadyPublished := ok && sv.Published
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
neverRetryFailures := []string{
"Error extracting sts from embedded url response",
@@ -1025,39 +1061,41 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
func (s *Sync) importPublicKey() error {
if s.publicKey != "" {
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accounts := accountsResponse.LBCMainnet
if logUtils.IsRegTest() {
accounts = accountsResponse.LBCRegtest
}
for _, a := range accounts {
if a.PublicKey == s.publicKey {
return nil
}
}
log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
_, err = s.daemon.AccountAdd(s.LbryChannelName, nil, nil, &s.publicKey, util.PtrToBool(true), nil)
return errors.Err(err)
}
return nil
}
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
func waitForDaemonProcess(timeout time.Duration) error {
processes, err := ps.Processes()
if err != nil {
return err
}
var daemonProcessId = -1
for _, p := range processes {
if p.Executable() == "lbrynet" {
daemonProcessId = p.Pid()
break
}
}
if daemonProcessId == -1 {
return nil
}
then := time.Now()
stopTime := then.Add(time.Duration(timeout * time.Second))
for !time.Now().After(stopTime) {
wait := 10 * time.Second
log.Println("the daemon is still running, waiting for it to exit")
time.Sleep(wait)
proc, err := os.FindProcess(daemonProcessId)
running, err := logUtils.IsLbrynetRunning()
if err != nil {
// couldn't find the process, that means the daemon is stopped and can continue
return nil
return errors.Err(err)
}
//double check if process is running and alive
//by sending a signal 0
//NOTE : syscall.Signal is not available in Windows
err = proc.Signal(syscall.Signal(0))
//the process doesn't exist anymore! we're free to go
if err != nil && (err == syscall.ESRCH || err.Error() == "os: process already finished") {
if !running {
return nil
}
}
+8 -1
View File
@@ -43,9 +43,16 @@ func (n *Namer) GetNextName(prefix string) string {
}
//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
attempt = 1
if len(name) < 2 {
sum := md5.Sum([]byte(prefix))
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
for {
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
if _, exists := n.names[name]; !exists {
break
}
attempt++
}
}
n.names[name] = true
+12
View File
@@ -34,6 +34,17 @@ type SyncProperties struct {
YoutubeChannelID string
}
type SyncFlags struct {
StopOnError bool
TakeOverExistingChannel bool
SkipSpaceCheck bool
SyncUpdate bool
SingleRun bool
RemoveDBUnpublished bool
UpgradeMetadata bool
DisableTransfers bool
}
type Fee struct {
Amount string `json:"amount"`
Address string `json:"address"`
@@ -47,6 +58,7 @@ type YoutubeChannel struct {
ChannelClaimID string `json:"channel_claim_id"`
TransferState int `json:"transfer_state"`
PublishAddress string `json:"publish_address"`
PublicKey string `json:"public_key"`
}
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
+9 -2
View File
@@ -519,8 +519,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
videoSize = uint64(existingVideoData.Size)
} else {
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
return nil, errors.Err("the video must be republished as we can't get the right size")
if !v.mocked {
_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
if err != nil {
return nil, errors.Err(err)
}
return v.downloadAndPublish(daemon, params)
}
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
}
}
v.size = util.PtrToInt64(int64(videoSize))