Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c28d5a716b | ||
|
|
59471f9c26 | ||
|
|
8028c6621c | ||
|
|
8abf6ad255 | ||
|
|
1f9a645c57 | ||
|
|
a5657aaf46 |
No files matched your search
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.48.1
|
||||
image: lbry/wallet-server:v0.50.1
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
|
||||
+12
-1
@@ -115,9 +115,20 @@ func (i *IPPool) ReleaseIP(ip string) {
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
|
||||
+9
-1
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
@@ -92,6 +93,7 @@ const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
@@ -159,7 +161,7 @@ func (s *SyncManager) Start() error {
|
||||
for _, q := range queuesToSync {
|
||||
//temporary override for sync-until to give tom the time to review the channels
|
||||
if q == StatusQueued {
|
||||
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
|
||||
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
|
||||
}
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
if err != nil {
|
||||
@@ -201,6 +203,12 @@ func (s *SyncManager) Start() error {
|
||||
shouldNotCount := false
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
if ipPool != nil {
|
||||
ipPool.ReleaseAll()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
|
||||
+5
-1
@@ -448,7 +448,11 @@ func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
addressResp, err := s.daemon.AddressUnused(nil)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if addressResp == nil {
|
||||
|
||||
+20
-4
@@ -246,6 +246,8 @@ func (s *Sync) setExceptions() {
|
||||
}
|
||||
}
|
||||
|
||||
var stopGroup = stop.New()
|
||||
|
||||
func (s *Sync) FullCycle() (e error) {
|
||||
if os.Getenv("HOME") == "" {
|
||||
return errors.Err("no $HOME env var found")
|
||||
@@ -258,7 +260,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stop.New()
|
||||
s.grp = stopGroup
|
||||
s.queue = make(chan video)
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
@@ -357,7 +359,18 @@ func (s *Sync) processTransfers() (e error) {
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
|
||||
}
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
@@ -777,7 +790,7 @@ func (s *Sync) doSync() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.transferState != TransferStateComplete {
|
||||
if s.transferState < TransferStateComplete {
|
||||
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel cert", err)
|
||||
@@ -883,6 +896,8 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -1073,7 +1088,7 @@ Enqueue:
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
log.Printf("stack: %s", debug.Stack())
|
||||
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
||||
var ok bool
|
||||
err, ok = p.(error)
|
||||
if !ok {
|
||||
@@ -1105,6 +1120,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
|
||||
log.Println(v.ID() + " can't ever be published")
|
||||
|
||||
+66
-67
@@ -199,7 +199,6 @@ func (v *YoutubeVideo) download() error {
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
qualityIndex := 0
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
@@ -224,94 +223,94 @@ func (v *YoutubeVideo) download() error {
|
||||
)
|
||||
}
|
||||
|
||||
sourceAddress, err := v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
var sourceAddress string
|
||||
for {
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err == nil {
|
||||
break
|
||||
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
runcmd:
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
for i, quality := range qualities {
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
|
||||
if i == (len(qualities) - 1) {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
break
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user