Compare commits

...
3 Commits
Author SHA1 Message Date
Niko Storni 52cd483f47 add single channelwide tag
fix bug
2019-06-10 21:37:13 +02:00
Niko Storni 79dd6d1458 add better handling for database mismatches
improve UTXOs management
fix bugs
2019-06-10 02:23:07 +02:00
Niko Storni 10ad46f464 various bug fixes 2019-06-06 23:25:31 +02:00
6 changed files with 54 additions and 26 deletions

No files matched your search

+3 -2
View File
@@ -2,8 +2,6 @@ package manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"strings"
"syscall"
"time"
@@ -13,6 +11,9 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
)
+5 -3
View File
@@ -2,6 +2,7 @@ package manager
import (
"fmt"
"math"
"net/http"
"os"
"strconv"
@@ -21,7 +22,7 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 3
const minimumRefillAmount = 4
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
@@ -189,9 +190,10 @@ func (s *Sync) ensureEnoughUTXOs() error {
broadcastFee := 0.01
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
desiredUTXOCount := uint64(math.Floor((balanceAmount - broadcastFee) / 0.1))
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, desiredUTXOCount)
if err != nil {
return err
} else if prefillTx == nil {
+38 -20
View File
@@ -444,6 +444,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
}
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
//additionally it removes all entries in the database indicating that a video is published when it's actually not
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
count := 0
videoIDMap := make(map[string]string, len(claims))
@@ -456,20 +457,39 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
videoIDMap[videoID] = c.ClaimID
pv, ok := s.syncedVideos[videoID]
if !ok || pv.ClaimName != c.Name {
fixed++
log.Debugf("adding %s to the database", c.Name)
size, err := c.GetStreamSizeByMagic()
pv, claimInDatabase := s.syncedVideos[videoID]
claimMetadataVersion := uint(1)
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
claimMetadataVersion = 2
}
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
claimMarkedUnpublished := claimInDatabase && !pv.Published
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
}
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
}
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
}
if claimMarkedUnpublished {
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
}
if !claimInDatabase {
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
claimSize, err := c.GetStreamSizeByMagic()
if err != nil {
size = 0
claimSize = 0
}
metadataVersion := uint(1)
claimIsUpgraded := strings.Contains(tn, thumbs.ThumbnailEndpoint)
if claimIsUpgraded {
metadataVersion = 2
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
fixed++
log.Debugf("updating %s in the database", videoID)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion)
if err != nil {
return count, fixed, 0, err
}
@@ -483,15 +503,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
idsToRemove = append(idsToRemove, vID)
}
}
removeCount := 0
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
if err != nil {
return count, fixed, 0, err
}
removeCount++
}
return count, fixed, removeCount, nil
return count, fixed, len(idsToRemove), nil
}
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
@@ -547,7 +565,7 @@ func (s *Sync) doSync() error {
return err
}
if nFixed > 0 {
SendInfoToSlack("%d claims were not on the remote database and were fixed", nFixed)
SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
}
if nRemoved > 0 {
SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
@@ -581,7 +599,7 @@ func (s *Sync) doSync() error {
}
if s.LbryChannelName == "@UCBerkeley" {
err = errors.Err("UCB is not supported on this version of YTSYNC")
err = errors.Err("UCB is not supported in this version of YTSYNC")
} else {
err = s.enqueueYoutubeVideos()
}
@@ -695,7 +713,7 @@ func (s *Sync) startWorker(workerNum int) {
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 0)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@@ -835,7 +853,7 @@ func (s *Sync) processVideo(v video) (err error) {
s.syncedVideosMux.RUnlock()
newMetadataVersion := int8(2)
alreadyPublished := ok && sv.Published
videoRequiresUpgrade := s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
neverRetryFailures := []string{
"Error extracting sts from embedded url response",
@@ -860,7 +878,7 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
if v.PlaylistPosition() > s.Manager.videosLimit {
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
log.Println(v.ID() + " is old: skipping")
return nil
}
+3 -1
View File
@@ -214,7 +214,9 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
if metadataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
}
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
}
+1
View File
@@ -471,6 +471,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err("the video must be republished as we can't get the right size")
}
}
v.size = util.PtrToInt64(int64(videoSize))
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
+4
View File
@@ -145,6 +145,8 @@ const (
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
DavidPakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
)
var channelWideTags = map[string][]string{
@@ -153,6 +155,8 @@ var channelWideTags = map[string][]string{
Juggling: {"juggling", "circus arts", "malabares"},
JustJuggling: {"juggling", "circus arts", "malabares"},
JordanBPeterson: {"postmodernism", "psychology", "news"},
DavidPakman: {"news"},
TheLinuxGamer: {"gaming", "technology", "linux"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,