Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24cf937e14 | ||
|
|
0b002c8228 | ||
|
|
a56166ee51 | ||
|
|
ddca850c17 | ||
|
|
31ce612e2e | ||
|
|
eb8900c66a | ||
|
|
67da4142d5 | ||
|
|
fb0e567caf |
No files matched your search
+96
-92
@@ -3,7 +3,6 @@ package downloader
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -25,8 +24,8 @@ import (
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
|
||||
ids, err := run(channelName, args, true, true, stopChan, pool)
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -44,8 +43,8 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
|
||||
results, err := run(videoID, args, true, true, stopChan, pool)
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
|
||||
results, err := run(videoID, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -126,7 +125,7 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
@@ -169,20 +168,25 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if release != nil {
|
||||
//const sqlTimeFormat = "2006-01-02 15:04:05"
|
||||
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
|
||||
if err == nil {
|
||||
return sqlTime.Format(releaseTimeFormat), nil
|
||||
} else {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
ytdlUploadDate, err := time.Parse("20060102", uploadDate)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if time.Now().AddDate(0, 0, -2).After(ytdlUploadDate) {
|
||||
if release != nil {
|
||||
//const sqlTimeFormat = "2006-01-02 15:04:05"
|
||||
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
|
||||
if err == nil {
|
||||
if sqlTime.Day() != ytdlUploadDate.Day() {
|
||||
logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
|
||||
} else {
|
||||
return sqlTime.Format(releaseTimeFormat), nil
|
||||
}
|
||||
} else {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), nil
|
||||
}
|
||||
client := getClient(ip)
|
||||
@@ -190,7 +194,7 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
@@ -244,96 +248,96 @@ func getClient(ip *net.TCPAddr) *http.Client {
|
||||
}
|
||||
}
|
||||
|
||||
func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var maxtries = 10
|
||||
var attemps int
|
||||
const (
|
||||
googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
|
||||
maxAttempts = 3
|
||||
extractionError = "YouTube said: Unable to extract video data"
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var useragent []string
|
||||
for {
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
sourceAddress, err := getIPFromPool(use, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer pool.ReleaseIP(sourceAddress)
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
cmd := exec.Command("youtube-dl", argsForCommand...)
|
||||
logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
|
||||
|
||||
var stderr io.ReadCloser
|
||||
var errorLog []byte
|
||||
if withStdErr {
|
||||
var err error
|
||||
stderr, err = cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
pool.ReleaseIP(sourceAddress)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
}
|
||||
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
//we don't want throttle errors to count toward the max retries
|
||||
attempts--
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, lastError
|
||||
}
|
||||
|
||||
var stdout io.ReadCloser
|
||||
var outLog []byte
|
||||
if withStdOut {
|
||||
var err error
|
||||
stdout, err = cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
func nextUA(current []string) []string {
|
||||
if len(current) == 0 {
|
||||
return []string{"--user-agent", googleBotUA}
|
||||
}
|
||||
return []string{"--user-agent", chromeUA}
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err = ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if withStdErr {
|
||||
errorLog, err = ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
}
|
||||
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
|
||||
var err error
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err := ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
errorLog, err := ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopChan:
|
||||
err := cmd.Process.Kill()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
attemps++
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
select {
|
||||
case <-stopChan:
|
||||
if err := cmd.Process.Kill(); err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
return nil, errors.Err("canceled by stopper")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
logrus.Debugf("known throttling error...try again (%d)", attemps)
|
||||
continue
|
||||
}
|
||||
if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
|
||||
useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
|
||||
if attemps == 1 {
|
||||
useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
|
||||
}
|
||||
logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attemps)
|
||||
continue
|
||||
}
|
||||
if attemps > maxtries {
|
||||
logrus.Debug("too many tries returning failure")
|
||||
break
|
||||
}
|
||||
}
|
||||
logrus.Debugf("Unkown error, returning failure: %s", err.Error())
|
||||
return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
|
||||
if len(errorLog) > 0 {
|
||||
return nil, errors.Err(string(errorLog))
|
||||
return nil, errors.Err("canceled by stopper")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+4
-1
@@ -71,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("cannot list claims", err)
|
||||
supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("cannot list supports", err)
|
||||
}
|
||||
}
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
|
||||
+6
-17
@@ -239,7 +239,7 @@ func deleteSyncFolder(videoDirectory string) {
|
||||
}
|
||||
|
||||
func (s *Sync) shouldTransfer() bool {
|
||||
return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers
|
||||
return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3
|
||||
}
|
||||
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
@@ -456,8 +456,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
|
||||
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
||||
_, isOwnClaim := ownClaimsInfo[videoID]
|
||||
tranferred := !isOwnClaim
|
||||
transferStatusMismatch := sv.Transferred != tranferred
|
||||
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
|
||||
transferStatusMismatch := sv.Transferred != transferred
|
||||
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
||||
@@ -475,7 +475,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
|
||||
}
|
||||
if transferStatusMismatch {
|
||||
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
|
||||
log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
@@ -493,7 +493,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
ClaimName: chainInfo.ClaimName,
|
||||
Size: util.PtrToInt64(int64(claimSize)),
|
||||
MetaDataVersion: chainInfo.MetadataVersion,
|
||||
IsTransferred: &tranferred,
|
||||
IsTransferred: &transferred,
|
||||
})
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
@@ -918,18 +918,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
|
||||
alreadyPublished := ok && sv.Published
|
||||
videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
|
||||
neverRetryFailures := []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
neverRetryFailures := shared.NeverRetryFailures
|
||||
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
|
||||
log.Println(v.ID() + " can't ever be published")
|
||||
return nil
|
||||
|
||||
+40
@@ -48,6 +48,11 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
|
||||
"channel_id": {cliFlags.ChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -106,6 +111,11 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
"auth_token": {a.ApiToken},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -149,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
}
|
||||
res, err := http.PostForm(endpoint, params)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -194,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
"channel_claim_id": {channelClaimID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -233,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -294,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -331,6 +361,11 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -380,6 +415,11 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
@@ -27,6 +27,19 @@ type YoutubeChannel struct {
|
||||
LastUploadedVideo string `json:"last_uploaded_video"`
|
||||
}
|
||||
|
||||
var NeverRetryFailures = []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
StopOnError bool
|
||||
TakeOverExistingChannel bool
|
||||
|
||||
+19
-4
@@ -209,6 +209,15 @@ func (v *YoutubeVideo) download() error {
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
|
||||
qualities = []string{
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
}
|
||||
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
@@ -219,12 +228,11 @@ func (v *YoutubeVideo) download() error {
|
||||
"-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"--user-agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--max-filesize",
|
||||
@@ -264,8 +272,10 @@ func (v *YoutubeVideo) download() error {
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
for i, quality := range qualities {
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
@@ -294,6 +304,11 @@ func (v *YoutubeVideo) download() error {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
|
||||
i-- //do not lower quality when trying a different user agent
|
||||
userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
|
||||
+13
-5
@@ -3,6 +3,7 @@ package ytapi
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -13,7 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
@@ -25,6 +26,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -55,16 +57,22 @@ type VideoParams struct {
|
||||
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||
|
||||
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
|
||||
|
||||
var videos []Video
|
||||
if quickSync && maxVideos > 50 {
|
||||
maxVideos = 50
|
||||
}
|
||||
videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
videoIDs := make([]string, 0, len(allVideos))
|
||||
for _, video := range allVideos {
|
||||
sv, ok := syncedVideos[video]
|
||||
if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
|
||||
continue
|
||||
}
|
||||
videoIDs = append(videoIDs, video)
|
||||
}
|
||||
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
|
||||
|
||||
playlistMap := make(map[string]int64)
|
||||
@@ -215,7 +223,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
|
||||
Status: "failed",
|
||||
FailureReason: err.Error(),
|
||||
})
|
||||
util.SendErrorToSlack("Skipping video: " + errors.FullTrace(err))
|
||||
logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
|
||||
if errSDK != nil {
|
||||
return nil, errors.Err(errSDK)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user