Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6944e17f43 | ||
|
|
a224fe44c2 | ||
|
|
fa9dc35123 | ||
|
|
3a0882230a | ||
|
|
fbd683e094 | ||
|
|
2f15c920d4 |
No files matched your search
@@ -29,7 +29,7 @@ import (
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
|
||||
@@ -20,7 +20,7 @@ require (
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
|
||||
@@ -283,8 +283,9 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b h1:/UnviTRIEwmkdLego+OI3Q75kKcztLuqPuJBYtcvz7w=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b/go.mod h1:K7IAMJMjDisynAcDkDdZHbZeWth6sTLl66xh+uJqtGg=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
|
||||
+8
-3
@@ -517,9 +517,14 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
claimSize := uint64(0)
|
||||
if chainInfo.Claim.Value.GetStream().Source != nil {
|
||||
claimSize, err = chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
}
|
||||
} else {
|
||||
util.SendToSlack("[%s] video with claimID %s has no source?! panic prevented...", s.DbChannelData.ChannelId, chainInfo.ClaimID)
|
||||
}
|
||||
fixed++
|
||||
log.Debugf("updating %s in the database", videoID)
|
||||
|
||||
+18
-7
@@ -216,9 +216,7 @@ func (v *YoutubeVideo) download() error {
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.IsLive != nil {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -276,6 +274,10 @@ func (v *YoutubeVideo) download() error {
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"--extractor-args",
|
||||
"youtube:player_client=android",
|
||||
//"--concurrent-fragments",
|
||||
//"2",
|
||||
"--load-info-json",
|
||||
metadataPath,
|
||||
}
|
||||
@@ -319,11 +321,15 @@ func (v *YoutubeVideo) download() error {
|
||||
sourceAddress,
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
|
||||
)
|
||||
|
||||
//speedThrottleRetries := 3
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
//if speedThrottleRetries > 0 {
|
||||
// speedThrottleRetries--
|
||||
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
|
||||
//}
|
||||
cmd := exec.Command("yt-dlp", argsWithFilters...)
|
||||
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
@@ -370,6 +376,10 @@ func (v *YoutubeVideo) download() error {
|
||||
userAgent = []string{downloader.GoogleBotUA}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
|
||||
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
|
||||
// i-- //do not lower quality when we're retrying a throttled download
|
||||
// continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
@@ -771,6 +781,9 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
minDuration := 7 * time.Second
|
||||
|
||||
if v.youtubeInfo.IsLive == true {
|
||||
return nil, errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
@@ -789,7 +802,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
break
|
||||
}
|
||||
|
||||
//log.Debugln("Downloaded " + v.id)
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
|
||||
@@ -885,9 +897,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
return nil, errors.Prefix("the video must be republished as we can't get the right size and it doesn't exist on youtube anymore", err)
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
|
||||
Reference in new issue
Block a user