Compare commits

..
4 Commits
Author SHA1 Message Date
Niko Storni c28d5a716b send warning to slack when video processing panics 2019-12-27 01:39:27 +01:00
Niko Storni 59471f9c26 reduce sync delay
fix stopper bug
2019-12-27 01:27:29 +01:00
Niko Storni 8028c6621c add temporary workaround to release all IPs
speed up one failure
2019-12-26 17:52:44 +01:00
Niko Storni 8abf6ad255 add debug statement 2019-12-24 23:27:16 +01:00
3 changed files with 25 additions and 4 deletions

No files matched your search

+12 -1
View File
@@ -115,9 +115,20 @@ func (i *IPPool) ReleaseIP(ip string) {
if localIP.IP == ip {
localIP.InUse = false
localIP.LastUse = time.Now()
break
return
}
}
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
}
func (i *IPPool) ReleaseAll() {
i.lock.Lock()
defer i.lock.Unlock()
for j, _ := range i.ips {
localIP := &i.ips[j]
localIP.InUse = false
localIP.LastUse = time.Now()
}
}
func (i *IPPool) SetThrottled(ip string) {
+8 -1
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/lbryio/ytsync/blobs_reflector"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
logUtils "github.com/lbryio/ytsync/util"
@@ -160,7 +161,7 @@ func (s *SyncManager) Start() error {
for _, q := range queuesToSync {
//temporary override for sync-until to give tom the time to review the channels
if q == StatusQueued {
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
}
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
if err != nil {
@@ -202,6 +203,12 @@ func (s *SyncManager) Start() error {
shouldNotCount := false
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
err := sync.FullCycle()
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
ipPool, _ := ip_manager.GetIPPool(sync.grp)
if ipPool != nil {
ipPool.ReleaseAll()
}
if err != nil {
fatalErrors := []string{
"default_wallet already exists",
+5 -2
View File
@@ -246,6 +246,8 @@ func (s *Sync) setExceptions() {
}
}
var stopGroup = stop.New()
func (s *Sync) FullCycle() (e error) {
if os.Getenv("HOME") == "" {
return errors.Err("no $HOME env var found")
@@ -258,7 +260,7 @@ func (s *Sync) FullCycle() (e error) {
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stop.New()
s.grp = stopGroup
s.queue = make(chan video)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
@@ -895,6 +897,7 @@ func (s *Sync) startWorker(workerNum int) {
"requested format not available",
"interrupted by user",
"Sign in to confirm your age",
"This video is unavailable",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -1085,7 +1088,7 @@ Enqueue:
func (s *Sync) processVideo(v video) (err error) {
defer func() {
if p := recover(); p != nil {
log.Printf("stack: %s", debug.Stack())
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
var ok bool
err, ok = p.(error)
if !ok {