Compare commits

...
4 Commits
Author SHA1 Message Date
Niko Storni 3a0882230a bypass throttling 2021-07-13 22:49:27 -04:00
Niko Storni fbd683e094 don't panic 2021-07-08 01:47:57 +02:00
Niko Storni 2f15c920d4 speed up playlist listing 2021-06-25 19:16:01 +02:00
Niko Storni 01f6448e72 revert aria2c usage 2021-06-25 19:09:00 +02:00
4 changed files with 24 additions and 17 deletions

No files matched your search

+1 -1
View File
@@ -29,7 +29,7 @@ import (
)
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
ids, err := run(channelName, args, stopChan, pool)
if err != nil {
return nil, errors.Err(err)
+1 -3
View File
@@ -2,7 +2,6 @@ package ip_manager
import (
"net"
"os"
"sort"
"sync"
"time"
@@ -46,8 +45,7 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
var pool []throttledIP
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
ipv6Disabled := os.Getenv("DISABLE_IPV6") != ""
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Disabled {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
+8 -3
View File
@@ -517,9 +517,14 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
if err != nil {
claimSize = 0
claimSize := uint64(0)
if chainInfo.Claim.Value.GetStream().Source != nil {
claimSize, err = chainInfo.Claim.GetStreamSizeByMagic()
if err != nil {
claimSize = 0
}
} else {
util.SendToSlack("[%s] video with claimID %s has no source?! panic prevented...", s.DbChannelData.ChannelId, chainInfo.ClaimID)
}
fixed++
log.Debugf("updating %s in the database", videoID)
+14 -10
View File
@@ -276,18 +276,14 @@ func (v *YoutubeVideo) download() error {
"1",
"--cookies",
"cookies.txt",
"--extractor-args",
"youtube:player_client=android",
//"--concurrent-fragments",
//"2",
"--load-info-json",
metadataPath,
}
ipv6Disabled := os.Getenv("DISABLE_IPV6") != ""
if ipv6Disabled {
ytdlArgs = append(ytdlArgs,
"--external-downloader",
"aria2c",
"--external-downloader-args",
"aria2c:-j 16 -x 16 -s 16 -k 1M",
)
}
userAgent := []string{"--user-agent", downloader.ChromeUA}
if v.maxVideoSize > 0 {
ytdlArgs = append(ytdlArgs,
@@ -327,11 +323,15 @@ func (v *YoutubeVideo) download() error {
sourceAddress,
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
)
//speedThrottleRetries := 3
for i := 0; i < len(qualities); i++ {
quality := qualities[i]
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
argsWithFilters = append(argsWithFilters, userAgent...)
//if speedThrottleRetries > 0 {
// speedThrottleRetries--
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
//}
cmd := exec.Command("yt-dlp", argsWithFilters...)
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
@@ -378,6 +378,10 @@ func (v *YoutubeVideo) download() error {
userAgent = []string{downloader.GoogleBotUA}
log.Infof("trying different user agent for video %s", v.ID())
continue
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
// i-- //do not lower quality when we're retrying a throttled download
// continue
}
return errors.Err(string(errorLog))
}