Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
52cd483f47 | ||
|
|
79dd6d1458 | ||
|
|
10ad46f464 |
No files matched your search
+3
-2
@@ -2,8 +2,6 @@ package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -13,6 +11,9 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
||||
+5
-3
@@ -2,6 +2,7 @@ package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
@@ -21,7 +22,7 @@ import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
const minimumRefillAmount = 3
|
||||
const minimumRefillAmount = 4
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
@@ -189,9 +190,10 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
broadcastFee := 0.01
|
||||
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
|
||||
|
||||
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
|
||||
desiredUTXOCount := uint64(math.Floor((balanceAmount - broadcastFee) / 0.1))
|
||||
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
|
||||
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, desiredUTXOCount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if prefillTx == nil {
|
||||
|
||||
+38
-20
@@ -444,6 +444,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
}
|
||||
|
||||
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
|
||||
//additionally it removes all entries in the database indicating that a video is published when it's actually not
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
|
||||
count := 0
|
||||
videoIDMap := make(map[string]string, len(claims))
|
||||
@@ -456,20 +457,39 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
videoIDMap[videoID] = c.ClaimID
|
||||
pv, ok := s.syncedVideos[videoID]
|
||||
if !ok || pv.ClaimName != c.Name {
|
||||
fixed++
|
||||
log.Debugf("adding %s to the database", c.Name)
|
||||
size, err := c.GetStreamSizeByMagic()
|
||||
pv, claimInDatabase := s.syncedVideos[videoID]
|
||||
claimMetadataVersion := uint(1)
|
||||
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
|
||||
claimMetadataVersion = 2
|
||||
}
|
||||
|
||||
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
|
||||
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
|
||||
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
|
||||
claimMarkedUnpublished := claimInDatabase && !pv.Published
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
|
||||
}
|
||||
if claimIDDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
|
||||
}
|
||||
if claimIDDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
|
||||
}
|
||||
if claimMarkedUnpublished {
|
||||
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
|
||||
}
|
||||
if !claimInDatabase {
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
|
||||
}
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
|
||||
claimSize, err := c.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
size = 0
|
||||
claimSize = 0
|
||||
}
|
||||
metadataVersion := uint(1)
|
||||
claimIsUpgraded := strings.Contains(tn, thumbs.ThumbnailEndpoint)
|
||||
if claimIsUpgraded {
|
||||
metadataVersion = 2
|
||||
}
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
|
||||
fixed++
|
||||
log.Debugf("updating %s in the database", videoID)
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion)
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
@@ -483,15 +503,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
}
|
||||
removeCount := 0
|
||||
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
|
||||
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
removeCount++
|
||||
}
|
||||
return count, fixed, removeCount, nil
|
||||
return count, fixed, len(idsToRemove), nil
|
||||
}
|
||||
|
||||
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
@@ -547,7 +565,7 @@ func (s *Sync) doSync() error {
|
||||
return err
|
||||
}
|
||||
if nFixed > 0 {
|
||||
SendInfoToSlack("%d claims were not on the remote database and were fixed", nFixed)
|
||||
SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
|
||||
}
|
||||
if nRemoved > 0 {
|
||||
SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
|
||||
@@ -581,7 +599,7 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
|
||||
if s.LbryChannelName == "@UCBerkeley" {
|
||||
err = errors.Err("UCB is not supported on this version of YTSYNC")
|
||||
err = errors.Err("UCB is not supported in this version of YTSYNC")
|
||||
} else {
|
||||
err = s.enqueueYoutubeVideos()
|
||||
}
|
||||
@@ -695,7 +713,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
}
|
||||
|
||||
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 0)
|
||||
if err != nil {
|
||||
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
}
|
||||
@@ -835,7 +853,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
s.syncedVideosMux.RUnlock()
|
||||
newMetadataVersion := int8(2)
|
||||
alreadyPublished := ok && sv.Published
|
||||
videoRequiresUpgrade := s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
|
||||
neverRetryFailures := []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
@@ -860,7 +878,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if v.PlaylistPosition() > s.Manager.videosLimit {
|
||||
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
|
||||
log.Println(v.ID() + " is old: skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
+3
-1
@@ -214,7 +214,9 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
|
||||
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
vals.Add("claim_id", claimID)
|
||||
vals.Add("claim_name", claimName)
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
|
||||
if metadataVersion > 0 {
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
|
||||
}
|
||||
if size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*size, 10))
|
||||
}
|
||||
|
||||
@@ -471,6 +471,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
|
||||
@@ -145,6 +145,8 @@ const (
|
||||
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
|
||||
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
|
||||
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
|
||||
DavidPakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
|
||||
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
|
||||
)
|
||||
|
||||
var channelWideTags = map[string][]string{
|
||||
@@ -153,6 +155,8 @@ var channelWideTags = map[string][]string{
|
||||
Juggling: {"juggling", "circus arts", "malabares"},
|
||||
JustJuggling: {"juggling", "circus arts", "malabares"},
|
||||
JordanBPeterson: {"postmodernism", "psychology", "news"},
|
||||
DavidPakman: {"news"},
|
||||
TheLinuxGamer: {"gaming", "technology", "linux"},
|
||||
}
|
||||
var tagsToSkip = map[string]*struct{}{
|
||||
"#hangoutsonair": nil,
|
||||
|
||||
Reference in new issue
Block a user