Compare commits

...
6 Commits
Author SHA1 Message Date
Niko Storni 4a356e86da add quicksync for better quota usage 2019-12-27 18:12:41 +01:00
Niko Storni ac95ca57af halt when quota is exceeded 2019-12-27 04:15:48 +01:00
Niko Storni c28d5a716b send warning to slack when video processing panics 2019-12-27 01:39:27 +01:00
Niko Storni 59471f9c26 reduce sync delay
fix stopper bug
2019-12-27 01:27:29 +01:00
Niko Storni 8028c6621c add temporary workaround to release all IPs
speed up one failure
2019-12-26 17:52:44 +01:00
Niko Storni 8abf6ad255 add debug statement 2019-12-24 23:27:16 +01:00
6 changed files with 32 additions and 9 deletions

No files matched your search

+2 -2
View File
@@ -53,7 +53,7 @@ echo "successfully started..."
./data_setup.sh
# Execute the sync test!
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
@@ -71,7 +71,7 @@ mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data
# Trigger transfer api
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
# Execute the transfer test!
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
+12 -1
View File
@@ -115,9 +115,20 @@ func (i *IPPool) ReleaseIP(ip string) {
if localIP.IP == ip {
localIP.InUse = false
localIP.LastUse = time.Now()
break
return
}
}
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
}
func (i *IPPool) ReleaseAll() {
i.lock.Lock()
defer i.lock.Unlock()
for j, _ := range i.ips {
localIP := &i.ips[j]
localIP.InUse = false
localIP.LastUse = time.Now()
}
}
func (i *IPPool) SetThrottled(ip string) {
+1
View File
@@ -56,6 +56,7 @@ func main() {
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
+9 -1
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/lbryio/ytsync/blobs_reflector"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
logUtils "github.com/lbryio/ytsync/util"
@@ -160,7 +161,7 @@ func (s *SyncManager) Start() error {
for _, q := range queuesToSync {
//temporary override for sync-until to give tom the time to review the channels
if q == StatusQueued {
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
}
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
if err != nil {
@@ -202,6 +203,12 @@ func (s *SyncManager) Start() error {
shouldNotCount := false
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
err := sync.FullCycle()
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
ipPool, _ := ip_manager.GetIPPool(sync.grp)
if ipPool != nil {
ipPool.ReleaseAll()
}
if err != nil {
fatalErrors := []string{
"default_wallet already exists",
@@ -212,6 +219,7 @@ func (s *SyncManager) Start() error {
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
"Daily Limit Exceeded",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
+7 -5
View File
@@ -246,6 +246,8 @@ func (s *Sync) setExceptions() {
}
}
var stopGroup = stop.New()
func (s *Sync) FullCycle() (e error) {
if os.Getenv("HOME") == "" {
return errors.Err("no $HOME env var found")
@@ -258,7 +260,7 @@ func (s *Sync) FullCycle() (e error) {
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stop.New()
s.grp = stopGroup
s.queue = make(chan video)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
@@ -895,6 +897,7 @@ func (s *Sync) startWorker(workerNum int) {
"requested format not available",
"interrupted by user",
"Sign in to confirm your age",
"This video is unavailable",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -1004,6 +1007,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
if err != nil {
return err
}
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
nextPageToken := ""
for {
@@ -1026,7 +1030,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
return errors.Err("playlist items not found")
}
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
videoIDs := make([]string, 50)
for i, item := range playlistResponse.Items {
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
@@ -1047,7 +1050,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
log.Infof("Got info for %d videos from youtube API", len(videos))
nextPageToken = playlistResponse.NextPageToken
if nextPageToken == "" {
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
break
}
}
@@ -1062,7 +1065,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
sort.Sort(byPublishedAt(videos))
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
Enqueue:
for _, v := range videos {
@@ -1085,7 +1087,7 @@ Enqueue:
func (s *Sync) processVideo(v video) (err error) {
defer func() {
if p := recover(); p != nil {
log.Printf("stack: %s", debug.Stack())
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
var ok bool
err, ok = p.(error)
if !ok {
+1
View File
@@ -45,6 +45,7 @@ type SyncFlags struct {
RemoveDBUnpublished bool
UpgradeMetadata bool
DisableTransfers bool
QuickSync bool
}
type Fee struct {