Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a56166ee51 | ||
|
|
ddca850c17 | ||
|
|
31ce612e2e | ||
|
|
eb8900c66a | ||
|
|
67da4142d5 | ||
|
|
fb0e567caf |
No files matched your search
+79
-80
@@ -3,7 +3,6 @@ package downloader
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -25,8 +24,8 @@ import (
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
|
||||
ids, err := run(channelName, args, true, true, stopChan, pool)
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -44,8 +43,8 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
|
||||
results, err := run(videoID, args, true, true, stopChan, pool)
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
|
||||
results, err := run(videoID, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -244,96 +243,96 @@ func getClient(ip *net.TCPAddr) *http.Client {
|
||||
}
|
||||
}
|
||||
|
||||
func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var maxtries = 10
|
||||
var attemps int
|
||||
const (
|
||||
googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"
|
||||
maxAttempts = 3
|
||||
extractionError = "YouTube said: Unable to extract video data"
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var useragent []string
|
||||
for {
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
sourceAddress, err := getIPFromPool(use, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer pool.ReleaseIP(sourceAddress)
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
cmd := exec.Command("youtube-dl", argsForCommand...)
|
||||
logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
|
||||
|
||||
var stderr io.ReadCloser
|
||||
var errorLog []byte
|
||||
if withStdErr {
|
||||
var err error
|
||||
stderr, err = cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
pool.ReleaseIP(sourceAddress)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
}
|
||||
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
//we don't want throttle errors to count toward the max retries
|
||||
attempts--
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, lastError
|
||||
}
|
||||
|
||||
var stdout io.ReadCloser
|
||||
var outLog []byte
|
||||
if withStdOut {
|
||||
var err error
|
||||
stdout, err = cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
func nextUA(current []string) []string {
|
||||
if len(current) == 0 {
|
||||
return []string{"--user-agent", googleBotUA}
|
||||
}
|
||||
return []string{"--user-agent", chromeUA}
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err = ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if withStdErr {
|
||||
errorLog, err = ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
}
|
||||
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
|
||||
var err error
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err := ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
errorLog, err := ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopChan:
|
||||
err := cmd.Process.Kill()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
attemps++
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
select {
|
||||
case <-stopChan:
|
||||
if err := cmd.Process.Kill(); err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
return nil, errors.Err("canceled by stopper")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
logrus.Debugf("known throttling error...try again (%d)", attemps)
|
||||
continue
|
||||
}
|
||||
if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
|
||||
useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
|
||||
if attemps == 1 {
|
||||
useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
|
||||
}
|
||||
logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attemps)
|
||||
continue
|
||||
}
|
||||
if attemps > maxtries {
|
||||
logrus.Debug("too many tries returning failure")
|
||||
break
|
||||
}
|
||||
}
|
||||
logrus.Debugf("Unkown error, returning failure: %s", err.Error())
|
||||
return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
|
||||
if len(errorLog) > 0 {
|
||||
return nil, errors.Err(string(errorLog))
|
||||
return nil, errors.Err("canceled by stopper")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+5
-5
@@ -239,7 +239,7 @@ func deleteSyncFolder(videoDirectory string) {
|
||||
}
|
||||
|
||||
func (s *Sync) shouldTransfer() bool {
|
||||
return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers
|
||||
return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3
|
||||
}
|
||||
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
@@ -456,8 +456,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
|
||||
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
||||
_, isOwnClaim := ownClaimsInfo[videoID]
|
||||
tranferred := !isOwnClaim
|
||||
transferStatusMismatch := sv.Transferred != tranferred
|
||||
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
|
||||
transferStatusMismatch := sv.Transferred != transferred
|
||||
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
||||
@@ -475,7 +475,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
|
||||
}
|
||||
if transferStatusMismatch {
|
||||
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
|
||||
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
@@ -493,7 +493,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
ClaimName: chainInfo.ClaimName,
|
||||
Size: util.PtrToInt64(int64(claimSize)),
|
||||
MetaDataVersion: chainInfo.MetadataVersion,
|
||||
IsTransferred: &tranferred,
|
||||
IsTransferred: &transferred,
|
||||
})
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
|
||||
+2
-1
@@ -3,6 +3,7 @@ package ytapi
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -215,7 +216,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
|
||||
Status: "failed",
|
||||
FailureReason: err.Error(),
|
||||
})
|
||||
util.SendErrorToSlack("Skipping video: " + errors.FullTrace(err))
|
||||
util.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
|
||||
if errSDK != nil {
|
||||
return nil, errors.Err(errSDK)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user