Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ad1704fb39 | ||
|
|
44e6cb5ddc | ||
|
|
5171acc007 | ||
|
|
946314da94 |
No files matched your search
@@ -12,7 +12,7 @@ require (
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/go-ini/ini v1.41.0 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/lbry.go v0.0.0-20190927133623-deab868c53ce
|
||||
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
|
||||
@@ -159,8 +159,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/Xo
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 h1:CyDDxUMREhAxPlgP+mgcArgkGJKtdXssj7CXk7o3h84=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190927133623-deab868c53ce h1:+irOdGmB0h9jNgL0PSFkCcds/Y9FWXicDpxRScxkJ+k=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190927133623-deab868c53ce/go.mod h1:wnoiUIfkeUmbiIGR/dg3Xvb6a6Jz7vPxtUoENyyl5QE=
|
||||
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85 h1:jBcKTwo0J10GzKBriDEUnT/we1edaaFZWtB7o57D9jI=
|
||||
github.com/lbryio/lbry.go v0.0.0-20191003205007-b1c090e28d85/go.mod h1:wnoiUIfkeUmbiIGR/dg3Xvb6a6Jz7vPxtUoENyyl5QE=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
|
||||
|
||||
+36
-36
@@ -126,24 +126,24 @@ func (s *SyncManager) Start() error {
|
||||
lbryChannelName := channels[0].DesiredChannelName
|
||||
syncs = make([]Sync, 1)
|
||||
syncs[0] = Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
publishAddress: channels[0].PublishAddress,
|
||||
publicKey: channels[0].PublicKey,
|
||||
transferState: channels[0].TransferState,
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
clientPublishAddress: channels[0].PublishAddress,
|
||||
publicKey: channels[0].PublicKey,
|
||||
transferState: channels[0].TransferState,
|
||||
}
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
@@ -169,24 +169,24 @@ func (s *SyncManager) Start() error {
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
publishAddress: c.PublishAddress,
|
||||
publicKey: c.PublicKey,
|
||||
transferState: c.TransferState,
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
clientPublishAddress: c.PublishAddress,
|
||||
publicKey: c.PublicKey,
|
||||
transferState: c.TransferState,
|
||||
})
|
||||
if q != StatusFailed {
|
||||
continue queues
|
||||
|
||||
+2
-5
@@ -131,7 +131,7 @@ func (s *Sync) walletSetup() error {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
if s.shouldTransfer() {
|
||||
s.claimAddress = s.publishAddress
|
||||
s.claimAddress = s.clientPublishAddress
|
||||
}
|
||||
|
||||
err = s.ensureEnoughUTXOs()
|
||||
@@ -277,10 +277,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if s.LbryChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
//@TODO: get rid of this when imported channels are supported
|
||||
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.transferState == TransferStateComplete {
|
||||
return nil
|
||||
}
|
||||
|
||||
+14
-7
@@ -11,10 +11,14 @@ import (
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allConfirmed := false
|
||||
waiting:
|
||||
for !allConfirmed {
|
||||
utxolist, err := s.daemon.UTXOList(nil)
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
@@ -57,7 +61,7 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
supportOnTransferredClaim := support.Address == s.publishAddress //todo: probably not needed anymore
|
||||
supportOnTransferredClaim := support.Address == s.clientPublishAddress //todo: probably not needed anymore
|
||||
if supportOnTransferredClaim {
|
||||
continue
|
||||
}
|
||||
@@ -83,7 +87,7 @@ func transferVideos(s *Sync) error {
|
||||
cleanTransfer := true
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
|
||||
log.Debugf("skipping video: %s", video.VideoID)
|
||||
//log.Debugf("skipping video: %s", video.VideoID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -98,9 +102,12 @@ func transferVideos(s *Sync) error {
|
||||
return errors.Err("cannot transfer: too many claims. claimID: %s", video.ClaimID)
|
||||
}
|
||||
|
||||
if c.Claims[0].Address == s.clientPublishAddress {
|
||||
continue
|
||||
}
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.publishAddress},
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
@@ -148,14 +155,14 @@ func transferChannel(s *Sync) error {
|
||||
if channelClaim == nil || len(channelClaim.Claims) == 0 {
|
||||
return errors.Err("There is no channel claim for channel %s", s.LbryChannelName)
|
||||
}
|
||||
if channelClaim.Claims[0].Address == s.publishAddress {
|
||||
if channelClaim.Claims[0].Address == s.clientPublishAddress {
|
||||
return nil
|
||||
}
|
||||
updateOptions := jsonrpc.ChannelUpdateOptions{
|
||||
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.publishAddress,
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -165,5 +172,5 @@ func transferChannel(s *Sync) error {
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
|
||||
|
||||
return errors.Err(err)
|
||||
return nil
|
||||
}
|
||||
+221
-101
@@ -8,6 +8,7 @@ import (
|
||||
"os/signal"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
@@ -62,32 +63,32 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
|
||||
|
||||
// Sync stores the options that control how syncing happens
|
||||
type Sync struct {
|
||||
APIConfig *sdk.APIConfig
|
||||
YoutubeChannelID string
|
||||
LbryChannelName string
|
||||
MaxTries int
|
||||
ConcurrentVideos int
|
||||
Refill int
|
||||
Manager *SyncManager
|
||||
LbrycrdString string
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
transferState int
|
||||
publishAddress string
|
||||
publicKey string
|
||||
APIConfig *sdk.APIConfig
|
||||
YoutubeChannelID string
|
||||
LbryChannelName string
|
||||
MaxTries int
|
||||
ConcurrentVideos int
|
||||
Refill int
|
||||
Manager *SyncManager
|
||||
LbrycrdString string
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
transferState int
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -314,44 +315,53 @@ func (s *Sync) FullCycle() (e error) {
|
||||
}
|
||||
|
||||
if s.shouldTransfer() {
|
||||
err := waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
supportAmount, err := abandonSupports(s)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
|
||||
}
|
||||
err = transferVideos(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = transferChannel(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reallocateSupports := supportAmount > 0.01
|
||||
if reallocateSupports {
|
||||
err = waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return s.processTransfers()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) processTransfers() (e error) {
|
||||
log.Println("Processing transfers")
|
||||
err := waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
supportAmount, err := abandonSupports(s)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
|
||||
}
|
||||
if supportAmount > 0 {
|
||||
logUtils.SendInfoToSlack("%.6f LBCs were abandoned and should be used as support", supportAmount)
|
||||
}
|
||||
err = transferVideos(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = transferChannel(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reallocateSupports := supportAmount > 0.01
|
||||
if reallocateSupports {
|
||||
err = waitConfirmations(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
}
|
||||
}
|
||||
log.Println("Done processing transfers")
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteSyncFolder(videoDirectory string) {
|
||||
if !strings.Contains(videoDirectory, "/tmp/ytsync") {
|
||||
_ = util.SendToSlack(errors.Err("Trying to delete an unexpected directory: %s", videoDirectory).Error())
|
||||
@@ -363,7 +373,7 @@ func deleteSyncFolder(videoDirectory string) {
|
||||
}
|
||||
|
||||
func (s *Sync) shouldTransfer() bool {
|
||||
return s.transferState >= 1 && s.publishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
|
||||
return s.transferState >= 1 && s.clientPublishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
|
||||
}
|
||||
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
@@ -495,7 +505,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
claimToAbandon = cl
|
||||
videoIDs[videoID] = c
|
||||
}
|
||||
if claimToAbandon.Address != s.publishAddress && !s.syncedVideos[videoID].Transferred {
|
||||
if claimToAbandon.Address != s.clientPublishAddress && !s.syncedVideos[videoID].Transferred {
|
||||
log.Debugf("abandoning %+v", claimToAbandon)
|
||||
_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
|
||||
if err != nil {
|
||||
@@ -510,52 +520,84 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
return abandonedClaims, nil
|
||||
}
|
||||
|
||||
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
|
||||
//additionally it removes all entries in the database indicating that a video is published when it's actually not
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
|
||||
count := 0
|
||||
videoIDMap := make(map[string]string, len(claims))
|
||||
type ytsyncClaim struct {
|
||||
ClaimID string
|
||||
MetadataVersion uint
|
||||
ClaimName string
|
||||
PublishAddress string
|
||||
VideoID string
|
||||
Claim *jsonrpc.Claim
|
||||
}
|
||||
|
||||
// mapFromClaims returns a map of videoIDs (youtube id) to ytsyncClaim which is a structure holding blockchain related
|
||||
// information
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
//check if claimID is in remote db
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
videoIDMap[videoID] = c.ClaimID
|
||||
s.syncedVideosMux.RLock()
|
||||
pv, claimInDatabase := s.syncedVideos[videoID]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
claimMetadataVersion := uint(1)
|
||||
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
|
||||
claimMetadataVersion = 2
|
||||
}
|
||||
|
||||
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
|
||||
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
|
||||
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
|
||||
claimMarkedUnpublished := claimInDatabase && !pv.Published
|
||||
videoIDMap[videoID] = ytsyncClaim{
|
||||
ClaimID: c.ClaimID,
|
||||
MetadataVersion: claimMetadataVersion,
|
||||
ClaimName: c.Name,
|
||||
PublishAddress: c.Address,
|
||||
VideoID: videoID,
|
||||
Claim: &c,
|
||||
}
|
||||
}
|
||||
return videoIDMap
|
||||
}
|
||||
|
||||
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
|
||||
//additionally it removes all entries in the database indicating that a video is published when it's actually not
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim) (total, fixed, removed int, err error) {
|
||||
allClaimsInfo := s.mapFromClaims(claims)
|
||||
ownClaimsInfo := s.mapFromClaims(ownClaims)
|
||||
count := len(allClaimsInfo)
|
||||
idsToRemove := make([]string, 0, count)
|
||||
|
||||
for videoID, chainInfo := range allClaimsInfo {
|
||||
s.syncedVideosMux.RLock()
|
||||
sv, claimInDatabase := s.syncedVideos[videoID]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
|
||||
metadataDiffers := claimInDatabase && sv.MetadataVersion != int8(chainInfo.MetadataVersion)
|
||||
claimIDDiffers := claimInDatabase && sv.ClaimID != chainInfo.ClaimID
|
||||
claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
|
||||
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
||||
_, isOwnClaim := ownClaimsInfo[videoID]
|
||||
tranferred := !isOwnClaim
|
||||
transferStatusMismatch := sv.Transferred != tranferred
|
||||
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
||||
}
|
||||
if claimIDDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
|
||||
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, sv.ClaimID, chainInfo.ClaimID)
|
||||
}
|
||||
if claimNameDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
|
||||
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, sv.ClaimName, chainInfo.ClaimName)
|
||||
}
|
||||
if claimMarkedUnpublished {
|
||||
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
|
||||
}
|
||||
if !claimInDatabase {
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
|
||||
}
|
||||
if s.syncedVideos[videoID].Transferred && s.publishAddress != c.Address {
|
||||
log.Debugf("%s: Marked as transferred while in fact it's not (%s - %s). Publish address: %s, expected: %s", videoID, c.Name, c.ClaimID, c.Address, s.publishAddress)
|
||||
if transferStatusMismatch {
|
||||
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
|
||||
}
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
|
||||
claimSize, err := c.GetStreamSizeByMagic()
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
}
|
||||
@@ -565,24 +607,36 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
||||
ChannelID: s.YoutubeChannelID,
|
||||
VideoID: videoID,
|
||||
Status: VideoStatusPublished,
|
||||
ClaimID: c.ClaimID,
|
||||
ClaimName: c.Name,
|
||||
ClaimID: chainInfo.ClaimID,
|
||||
ClaimName: chainInfo.ClaimName,
|
||||
Size: util.PtrToInt64(int64(claimSize)),
|
||||
MetaDataVersion: claimMetadataVersion,
|
||||
IsTransferred: util.PtrToBool(s.publishAddress == c.Address),
|
||||
MetaDataVersion: chainInfo.MetadataVersion,
|
||||
IsTransferred: &tranferred,
|
||||
})
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
idsToRemove := make([]string, 0, len(videoIDMap))
|
||||
|
||||
//reload the synced videos map before we use it for further processing
|
||||
if fixed > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
for vID, sv := range s.syncedVideos {
|
||||
if sv.Transferred {
|
||||
log.Infof("%s: claim was transferred, ignoring", vID)
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
_, ok := videoIDMap[vID]
|
||||
_, ok := ownClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
@@ -594,15 +648,31 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
||||
if err != nil {
|
||||
return count, fixed, len(idsToRemove), err
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return count, fixed, 0, nil
|
||||
//reload the synced videos map before we use it for further processing
|
||||
if removed > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return count, fixed, removed, err
|
||||
}
|
||||
}
|
||||
return count, fixed, removed, nil
|
||||
}
|
||||
|
||||
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
totalPages := uint64(1)
|
||||
var allClaims []jsonrpc.Claim
|
||||
var account *string = nil
|
||||
if defaultOnly {
|
||||
a, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
account = &a
|
||||
}
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
claims, err := s.daemon.ClaimList(nil, page, 50)
|
||||
claims, err := s.daemon.ClaimList(account, page, 50)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
@@ -613,7 +683,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
allClaims, err := s.getClaims()
|
||||
allClaims, err := s.getClaims(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -627,22 +697,22 @@ func (s *Sync) checkIntegrity() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allClaims, err = s.getClaims()
|
||||
allClaims, err = s.getClaims(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
pubsOnWallet, nFixed, nRemoved, err := s.updateRemoteDB(allClaims)
|
||||
ownClaims, err := s.getClaims(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pubsOnWallet, nFixed, nRemoved, err := s.updateRemoteDB(allClaims, ownClaims)
|
||||
if err != nil {
|
||||
return errors.Prefix("error updating remote database", err)
|
||||
}
|
||||
|
||||
if nFixed > 0 || nRemoved > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if nFixed > 0 {
|
||||
logUtils.SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
|
||||
}
|
||||
@@ -664,6 +734,11 @@ func (s *Sync) checkIntegrity() error {
|
||||
if pubsOnWallet < pubsOnDB {
|
||||
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
_, err = s.getUnsentSupports() //TODO: use the returned value when it works
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1083,6 +1158,51 @@ func (s *Sync) importPublicKey() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//TODO: fully implement this once I find a way to reliably get the abandoned supports amount
|
||||
func (s *Sync) getUnsentSupports() (float64, error) {
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if s.transferState == 2 {
|
||||
balance, err := s.daemon.AccountBalance(&defaultAccount)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
} else if balance == nil {
|
||||
return 0, errors.Err("no response")
|
||||
}
|
||||
|
||||
balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
transactionList, err := s.daemon.TransactionList(&defaultAccount)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
sentSupports := 0.0
|
||||
for _, t := range *transactionList {
|
||||
if len(t.SupportInfo) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, support := range t.SupportInfo {
|
||||
supportAmount, err := strconv.ParseFloat(support.BalanceDelta, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if supportAmount < 0 { // && support.IsTip TODO: re-enable this when transaction list shows correct information
|
||||
sentSupports += -supportAmount
|
||||
}
|
||||
}
|
||||
}
|
||||
if balanceAmount > 10 && sentSupports < 1 {
|
||||
logUtils.SendErrorToSlack("this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", balanceAmount, sentSupports)
|
||||
return balanceAmount - 10, nil
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
|
||||
func waitForDaemonProcess(timeout time.Duration) error {
|
||||
then := time.Now()
|
||||
|
||||
Reference in new issue
Block a user