Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3a0882230a | ||
|
|
fbd683e094 | ||
|
|
2f15c920d4 | ||
|
|
01f6448e72 |
No files matched your search
@@ -29,7 +29,7 @@ import (
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
|
||||
@@ -2,7 +2,6 @@ package ip_manager
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -46,8 +45,7 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
var pool []throttledIP
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
ipv6Disabled := os.Getenv("DISABLE_IPV6") != ""
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Disabled {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
|
||||
+8
-3
@@ -517,9 +517,14 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
claimSize := uint64(0)
|
||||
if chainInfo.Claim.Value.GetStream().Source != nil {
|
||||
claimSize, err = chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
}
|
||||
} else {
|
||||
util.SendToSlack("[%s] video with claimID %s has no source?! panic prevented...", s.DbChannelData.ChannelId, chainInfo.ClaimID)
|
||||
}
|
||||
fixed++
|
||||
log.Debugf("updating %s in the database", videoID)
|
||||
|
||||
+14
-10
@@ -276,18 +276,14 @@ func (v *YoutubeVideo) download() error {
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"--extractor-args",
|
||||
"youtube:player_client=android",
|
||||
//"--concurrent-fragments",
|
||||
//"2",
|
||||
"--load-info-json",
|
||||
metadataPath,
|
||||
}
|
||||
ipv6Disabled := os.Getenv("DISABLE_IPV6") != ""
|
||||
if ipv6Disabled {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--external-downloader",
|
||||
"aria2c",
|
||||
"--external-downloader-args",
|
||||
"aria2c:-j 16 -x 16 -s 16 -k 1M",
|
||||
)
|
||||
}
|
||||
|
||||
userAgent := []string{"--user-agent", downloader.ChromeUA}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
@@ -327,11 +323,15 @@ func (v *YoutubeVideo) download() error {
|
||||
sourceAddress,
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
|
||||
)
|
||||
|
||||
//speedThrottleRetries := 3
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
//if speedThrottleRetries > 0 {
|
||||
// speedThrottleRetries--
|
||||
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
|
||||
//}
|
||||
cmd := exec.Command("yt-dlp", argsWithFilters...)
|
||||
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
@@ -378,6 +378,10 @@ func (v *YoutubeVideo) download() error {
|
||||
userAgent = []string{downloader.GoogleBotUA}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
|
||||
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
|
||||
// i-- //do not lower quality when we're retrying a throttled download
|
||||
// continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user