Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6861dae9b | ||
|
|
cff147bb95 | ||
|
|
54d04dcf2c |
No files matched your search
+41
-16
@@ -19,20 +19,22 @@ const unbanTimeout = 3 * time.Hour
|
||||
var stopper = stop.New()
|
||||
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.Mutex
|
||||
ips []throttledIP
|
||||
lock *sync.RWMutex
|
||||
stopGrp *stop.Group
|
||||
}
|
||||
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
IP string
|
||||
UsedForVideo string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool() (*IPPool, error) {
|
||||
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
@@ -59,9 +61,25 @@ func GetIPPool() (*IPPool, error) {
|
||||
}
|
||||
}
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.Mutex{},
|
||||
ips: pool,
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopGrp.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ipPoolInstance.lock.RLock()
|
||||
for _, ip := range ipPoolInstance.ips {
|
||||
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
}
|
||||
ipPoolInstance.lock.RUnlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
@@ -102,7 +120,7 @@ func (i *IPPool) ReleaseIP(ip string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
@@ -129,7 +147,7 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
case <-i.stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}(tIP)
|
||||
@@ -138,8 +156,9 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
@@ -165,18 +184,24 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
nextIP.UsedForVideo = forVideo
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP() (string, error) {
|
||||
func (i *IPPool) GetIP(forVideo string) (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP()
|
||||
ip, err := i.nextIP(forVideo)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
select {
|
||||
case <-i.stopGrp.Ch():
|
||||
return "", errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
|
||||
@@ -399,6 +399,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
if channelInfo.DefaultLanguage == "iw" {
|
||||
channelInfo.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
var locations []jsonrpc.Location = nil
|
||||
|
||||
+18
-5
@@ -638,8 +638,17 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
if sv.Transferred {
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
continue
|
||||
}
|
||||
if len(searchResponse.Claims) == 0 {
|
||||
log.Debugf("%s: was transferred but appears abandoned! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
continue //TODO: we should flag these on the db
|
||||
} else {
|
||||
return count, fixed, 0, errors.Err("%s: isn't our control but is on the database and on the blockchain. wtf is up? ClaimID: %s", vID, sv.ClaimID)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -873,12 +882,15 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
} else if tryCount < s.MaxTries {
|
||||
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
|
||||
strings.Contains(err.Error(), "too-long-mempool-chain") {
|
||||
if util.SubstringInSlice(err.Error(), []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -890,6 +902,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
"Missing inputs",
|
||||
}) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
@@ -975,7 +988,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
|
||||
var videos []video
|
||||
ipPool, err := ip_manager.GetIPPool()
|
||||
ipPool, err := ip_manager.GetIPPool(s.grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -224,7 +224,7 @@ func (v *YoutubeVideo) download() error {
|
||||
)
|
||||
}
|
||||
|
||||
sourceAddress, err := v.pool.GetIP()
|
||||
sourceAddress, err := v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
@@ -235,7 +235,7 @@ func (v *YoutubeVideo) download() error {
|
||||
}
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = v.pool.GetIP()
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err == nil {
|
||||
break
|
||||
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
@@ -278,7 +278,7 @@ runcmd:
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress, v.stopGroup)
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
@@ -459,6 +459,9 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user