Compare commits

...
8 Commits
Author SHA1 Message Date
Niko Storni 24cf937e14 retry on support list failures 2020-08-18 01:09:53 +02:00
Niko Storni 0b002c8228 lots of bug fixes and improvements 2020-08-18 00:03:38 +02:00
Niko Storni a56166ee51 refactor youtube-dl execution process 2020-08-12 19:44:57 +02:00
Niko Storni ddca850c17 nevermind... re-enable ipv6 (i should make it a flag) 2020-08-12 04:17:30 +02:00
Niko Storni 31ce612e2e temporarily disable ipv6 2020-08-12 03:38:52 +02:00
Niko Storni eb8900c66a don't waste loops
fix a deadlock?
2020-08-12 03:37:18 +02:00
Niko Storni 67da4142d5 fix old transferred channels 2020-08-10 18:33:05 +02:00
Niko Storni fb0e567caf be more specific on video failures 2020-08-08 03:29:57 +02:00
7 changed files with 191 additions and 119 deletions

No files matched your search

+96 -92
View File
@@ -3,7 +3,6 @@ package downloader
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
@@ -25,8 +24,8 @@ import (
)
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
ids, err := run(channelName, args, true, true, stopChan, pool)
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
ids, err := run(channelName, args, stopChan, pool)
if err != nil {
return nil, errors.Err(err)
}
@@ -44,8 +43,8 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
results, err := run(videoID, args, true, true, stopChan, pool)
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
results, err := run(videoID, args, stopChan, pool)
if err != nil {
return nil, errors.Err(err)
}
@@ -126,7 +125,7 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
if err != nil {
return errors.Err(err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
res, err := client.Do(req)
if err != nil {
@@ -169,20 +168,25 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
if err != nil {
logrus.Error(err)
}
if release != nil {
//const sqlTimeFormat = "2006-01-02 15:04:05"
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
if err == nil {
return sqlTime.Format(releaseTimeFormat), nil
} else {
logrus.Error(err)
}
}
ytdlUploadDate, err := time.Parse("20060102", uploadDate)
if err != nil {
logrus.Error(err)
}
if time.Now().AddDate(0, 0, -2).After(ytdlUploadDate) {
if release != nil {
//const sqlTimeFormat = "2006-01-02 15:04:05"
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
if err == nil {
if sqlTime.Day() != ytdlUploadDate.Day() {
logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
} else {
return sqlTime.Format(releaseTimeFormat), nil
}
} else {
logrus.Error(err)
}
}
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
return ytdlUploadDate.Format(releaseTimeFormat), nil
}
client := getClient(ip)
@@ -190,7 +194,7 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
res, err := client.Do(req)
if err != nil {
@@ -244,96 +248,96 @@ func getClient(ip *net.TCPAddr) *http.Client {
}
}
func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
var maxtries = 10
var attemps int
const (
googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
maxAttempts = 3
extractionError = "YouTube said: Unable to extract video data"
throttledError = "HTTP Error 429"
AlternateThrottledError = "returned non-zero exit status 8"
youtubeDlError = "exit status 1"
)
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
var useragent []string
for {
var lastError error
for attempts := 0; attempts < maxAttempts; attempts++ {
sourceAddress, err := getIPFromPool(use, stopChan, pool)
if err != nil {
return nil, err
}
defer pool.ReleaseIP(sourceAddress)
argsForCommand := append(args, "--source-address", sourceAddress)
argsForCommand = append(argsForCommand, useragent...)
cmd := exec.Command("youtube-dl", argsForCommand...)
logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
var stderr io.ReadCloser
var errorLog []byte
if withStdErr {
var err error
stderr, err = cmd.StderrPipe()
if err != nil {
return nil, errors.Err(err)
res, err := runCmd(cmd, stopChan)
pool.ReleaseIP(sourceAddress)
if err == nil {
return res, nil
}
lastError = err
if strings.Contains(err.Error(), youtubeDlError) {
if strings.Contains(err.Error(), extractionError) {
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
useragent = nextUA(useragent)
}
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
pool.SetThrottled(sourceAddress)
//we don't want throttle errors to count toward the max retries
attempts--
}
}
}
return nil, lastError
}
var stdout io.ReadCloser
var outLog []byte
if withStdOut {
var err error
stdout, err = cmd.StdoutPipe()
if err != nil {
return nil, errors.Err(err)
}
func nextUA(current []string) []string {
if len(current) == 0 {
return []string{"--user-agent", googleBotUA}
}
return []string{"--user-agent", chromeUA}
}
if err := cmd.Start(); err != nil {
return nil, errors.Err(err)
}
outLog, err = ioutil.ReadAll(stdout)
if err != nil {
return nil, errors.Err(err)
}
if withStdErr {
errorLog, err = ioutil.ReadAll(stderr)
if err != nil {
return nil, errors.Err(err)
}
}
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
var err error
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, errors.Err(err)
}
err = cmd.Start()
if err != nil {
return nil, errors.Err(err)
}
outLog, err := ioutil.ReadAll(stdout)
if err != nil {
return nil, errors.Err(err)
}
errorLog, err := ioutil.ReadAll(stderr)
if err != nil {
return nil, errors.Err(err)
}
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
select {
case <-stopChan:
err := cmd.Process.Kill()
if err != nil {
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
}
done := make(chan error, 1)
go func() {
attemps++
done <- cmd.Wait()
}()
select {
case <-stopChan:
if err := cmd.Process.Kill(); err != nil {
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
}
return nil, errors.Err("canceled by stopper")
case err := <-done:
if err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
pool.SetThrottled(sourceAddress)
logrus.Debugf("known throttling error...try again (%d)", attemps)
continue
}
if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
if attemps == 1 {
useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
}
logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attemps)
continue
}
if attemps > maxtries {
logrus.Debug("too many tries returning failure")
break
}
}
logrus.Debugf("Unkown error, returning failure: %s", err.Error())
return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
}
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
}
if len(errorLog) > 0 {
return nil, errors.Err(string(errorLog))
return nil, errors.Err("canceled by stopper")
case err := <-done:
if err != nil {
return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
}
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
}
}
+4 -1
View File
@@ -71,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) {
for page := uint64(1); page <= totalPages; page++ {
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
if err != nil {
return 0, errors.Prefix("cannot list claims", err)
supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
if err != nil {
return 0, errors.Prefix("cannot list supports", err)
}
}
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
+6 -17
View File
@@ -239,7 +239,7 @@ func deleteSyncFolder(videoDirectory string) {
}
func (s *Sync) shouldTransfer() bool {
return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers
return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3
}
func (s *Sync) setChannelTerminationStatus(e *error) {
@@ -456,8 +456,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
claimMarkedUnpublished := claimInDatabase && !sv.Published
_, isOwnClaim := ownClaimsInfo[videoID]
tranferred := !isOwnClaim
transferStatusMismatch := sv.Transferred != tranferred
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
transferStatusMismatch := sv.Transferred != transferred
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -475,7 +475,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
}
if transferStatusMismatch {
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
@@ -493,7 +493,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
ClaimName: chainInfo.ClaimName,
Size: util.PtrToInt64(int64(claimSize)),
MetaDataVersion: chainInfo.MetadataVersion,
IsTransferred: &tranferred,
IsTransferred: &transferred,
})
if err != nil {
return count, fixed, 0, err
@@ -918,18 +918,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
alreadyPublished := ok && sv.Published
videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
neverRetryFailures := []string{
"Error extracting sts from embedded url response",
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
"This video contains content from",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"giving up after 0 fragment retries",
"Sign in to confirm your age",
}
neverRetryFailures := shared.NeverRetryFailures
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
log.Println(v.ID() + " can't ever be published")
return nil
+40
View File
@@ -48,6 +48,11 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
"channel_id": {cliFlags.ChannelID},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cliFlags)
}
return nil, errors.Err(err)
}
defer res.Body.Close()
@@ -106,6 +111,11 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
"auth_token": {a.ApiToken},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
return errors.Err(err)
}
defer res.Body.Close()
@@ -149,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
}
res, err := http.PostForm(endpoint, params)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
return nil, nil, errors.Err(err)
}
defer res.Body.Close()
@@ -194,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
"channel_claim_id": {channelClaimID},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
return errors.Err(err)
}
defer res.Body.Close()
@@ -233,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
return errors.Err(err)
}
defer res.Body.Close()
@@ -294,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
return errors.Err(err)
}
defer res.Body.Close()
@@ -331,6 +361,11 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.VideoState(videoID)
}
return "", errors.Err(err)
}
defer res.Body.Close()
@@ -380,6 +415,11 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.GetReleasedDate(videoID)
}
return nil, errors.Err(err)
}
defer res.Body.Close()
+13
View File
@@ -27,6 +27,19 @@ type YoutubeChannel struct {
LastUploadedVideo string `json:"last_uploaded_video"`
}
var NeverRetryFailures = []string{
"Error extracting sts from embedded url response",
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
"This video contains content from",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"giving up after 0 fragment retries",
"Sign in to confirm your age",
}
type SyncFlags struct {
StopOnError bool
TakeOverExistingChannel bool
+19 -4
View File
@@ -209,6 +209,15 @@ func (v *YoutubeVideo) download() error {
"480",
"320",
}
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
qualities = []string{
"720",
"480",
"320",
}
}
ytdlArgs := []string{
"--no-progress",
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
@@ -219,12 +228,11 @@ func (v *YoutubeVideo) download() error {
"-movflags faststart",
"--abort-on-unavailable-fragment",
"--fragment-retries",
"0",
"--user-agent",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
"1",
"--cookies",
"cookies.txt",
}
userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
if v.maxVideoSize > 0 {
ytdlArgs = append(ytdlArgs,
"--max-filesize",
@@ -264,8 +272,10 @@ func (v *YoutubeVideo) download() error {
"https://www.youtube.com/watch?v="+v.ID(),
)
for i, quality := range qualities {
for i := 0; i < len(qualities); i++ {
quality := qualities[i]
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
argsWithFilters = append(argsWithFilters, userAgent...)
cmd := exec.Command("youtube-dl", argsWithFilters...)
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
@@ -294,6 +304,11 @@ func (v *YoutubeVideo) download() error {
return errors.Err(string(errorLog))
}
continue //this bypasses the yt throttling IP redistribution... TODO: don't
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
i-- //do not lower quality when trying a different user agent
userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
log.Infof("trying different user agent for video %s", v.ID())
continue
}
return errors.Err(string(errorLog))
}
+13 -5
View File
@@ -3,6 +3,7 @@ package ytapi
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"regexp"
@@ -13,7 +14,7 @@ import (
"time"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/util"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
@@ -25,6 +26,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/aws/aws-sdk-go/aws"
log "github.com/sirupsen/logrus"
@@ -55,16 +57,22 @@ type VideoParams struct {
var mostRecentlyFailedChannel string // TODO: fix this hack!
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
var videos []Video
if quickSync && maxVideos > 50 {
maxVideos = 50
}
videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
if err != nil {
return nil, errors.Err(err)
}
videoIDs := make([]string, 0, len(allVideos))
for _, video := range allVideos {
sv, ok := syncedVideos[video]
if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
continue
}
videoIDs = append(videoIDs, video)
}
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
playlistMap := make(map[string]int64)
@@ -215,7 +223,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
Status: "failed",
FailureReason: err.Error(),
})
util.SendErrorToSlack("Skipping video: " + errors.FullTrace(err))
logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
if errSDK != nil {
return nil, errors.Err(errSDK)
}