Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e554bbfe18 | ||
|
|
6944e17f43 | ||
|
|
a224fe44c2 | ||
|
|
fa9dc35123 | ||
|
|
3a0882230a | ||
|
|
fbd683e094 | ||
|
|
2f15c920d4 | ||
|
|
01f6448e72 | ||
|
|
8fb1e2ead0 | ||
|
|
3b84db382c | ||
|
|
4fe6840a4e | ||
|
|
69e6fb51d1 | ||
|
|
f17110ab7f |
No files matched your search
@@ -1,111 +0,0 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package cmd
|
||||
|
||||
import "github.com/lbryio/ytsync/v5/local"
|
||||
|
||||
func init() {
|
||||
local.AddCommand(rootCmd)
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
Run: ytSync,
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
rootCmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
rootCmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
rootCmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
|
||||
rootCmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
rootCmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
rootCmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
rootCmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
rootCmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
|
||||
rootCmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
rootCmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func ytSync(cmd *cobra.Command, args []string) {
|
||||
var hostname string
|
||||
slackToken := os.Getenv("SLACK_TOKEN")
|
||||
if slackToken == "" {
|
||||
log.Error("A slack token was not present in env vars! Slack messages disabled!")
|
||||
} else {
|
||||
var err error
|
||||
hostname, err = os.Hostname()
|
||||
if err != nil {
|
||||
log.Error("could not detect system hostname")
|
||||
hostname = "ytsync-unknown"
|
||||
}
|
||||
if len(hostname) > 30 {
|
||||
hostname = hostname[0:30]
|
||||
}
|
||||
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
|
||||
if apiURL == "" {
|
||||
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
|
||||
return
|
||||
}
|
||||
if apiToken == "" {
|
||||
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
|
||||
return
|
||||
}
|
||||
if youtubeAPIKey == "" {
|
||||
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
|
||||
return
|
||||
}
|
||||
if awsS3ID == "" {
|
||||
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
|
||||
return
|
||||
}
|
||||
if awsS3Secret == "" {
|
||||
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
|
||||
return
|
||||
}
|
||||
if awsS3Region == "" {
|
||||
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
|
||||
return
|
||||
}
|
||||
if awsS3Bucket == "" {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
)
|
||||
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
ytUtils.SendErrorToSlack(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
ytUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
util2 "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
@@ -28,8 +29,8 @@ import (
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool, true)
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -56,7 +57,7 @@ func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Ch
|
||||
"-o",
|
||||
path.Join(util2.GetVideoMetadataDir(), videoID),
|
||||
}
|
||||
_, err := run(videoID, args, stopChan, pool, false)
|
||||
_, err := run(videoID, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -279,9 +280,11 @@ const (
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
videoPremiereError = "Premieres in"
|
||||
liveEventError = "This live event will begin in"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool, dlc bool) ([]string, error) {
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var useragent []string
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
@@ -292,9 +295,6 @@ func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool,
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
binary := "yt-dlp"
|
||||
if dlc {
|
||||
binary = "yt-dlp"
|
||||
}
|
||||
cmd := exec.Command(binary, argsForCommand...)
|
||||
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
@@ -304,6 +304,9 @@ func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool,
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if util.SubstringInSlice(err.Error(), shared.ErrorsNoRetry) {
|
||||
break
|
||||
}
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
|
||||
@@ -20,7 +20,7 @@ require (
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
|
||||
@@ -283,8 +283,9 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b h1:/UnviTRIEwmkdLego+OI3Q75kKcztLuqPuJBYtcvz7w=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b/go.mod h1:K7IAMJMjDisynAcDkDdZHbZeWth6sTLl66xh+uJqtGg=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func AddCommand(rootCmd *cobra.Command) {
|
||||
cmd := &cobra.Command{
|
||||
Use: "local",
|
||||
Short: "run a personal ytsync",
|
||||
Run: localCmd,
|
||||
}
|
||||
//cmd.Flags().StringVar(&cache, "cache", "", "path to cache")
|
||||
rootCmd.AddCommand(cmd)
|
||||
|
||||
}
|
||||
|
||||
func localCmd(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("local")
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
# Running ytsync locally
|
||||
|
||||
## Requirements
|
||||
|
||||
- LBRY SDK (what do we actually need this for?)
|
||||
- youtube-dl
|
||||
- enough space to cache stuff
|
||||
|
||||
|
||||
## Process
|
||||
|
||||
### Ensuring requirements are met
|
||||
|
||||
- claim channel if there isn't one yet
|
||||
- or easier, just error if no channel
|
||||
- enough lbc in wallet?
|
||||
|
||||
### Options to figure out what's already synced
|
||||
|
||||
- simplest: assume nothing is synced yet
|
||||
- assume everything before some video is synced
|
||||
- get/put sync info from Odysee by proving you have private key for channel
|
||||
- tag videos as having been synced from youtube so we can ensure accuracy
|
||||
- hardest: scan channel and try to match up which videos are not synced yet
|
||||
|
||||
### Central DB
|
||||
|
||||
- prove you have a channel's private key to get info about that channel
|
||||
- proper queue instead of sleeping for N minutes between syncs
|
||||
|
||||
|
||||
|
||||
### Syncing a single video
|
||||
|
||||
- downloading it
|
||||
- thumbnails
|
||||
- metadata
|
||||
- having enough LBC for publish(es)
|
||||
- automated error handling
|
||||
- getting a human involved for errors that can't be handled automatically
|
||||
- reflecting
|
||||
|
||||
### Continuous Sync
|
||||
|
||||
- running in background
|
||||
- storing local state
|
||||
- interactions with our central ytsync db
|
||||
- dealing with yt throttling
|
||||
|
||||
|
||||
### Debugging
|
||||
|
||||
- dry-running the whole thing
|
||||
@@ -1,172 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/cmd"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var Version string
|
||||
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
go func() {
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
}()
|
||||
cmd := &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
Run: ytSync,
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
|
||||
cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func ytSync(cmd *cobra.Command, args []string) {
|
||||
var hostname string
|
||||
slackToken := os.Getenv("SLACK_TOKEN")
|
||||
if slackToken == "" {
|
||||
log.Error("A slack token was not present in env vars! Slack messages disabled!")
|
||||
} else {
|
||||
var err error
|
||||
hostname, err = os.Hostname()
|
||||
if err != nil {
|
||||
log.Error("could not detect system hostname")
|
||||
hostname = "ytsync-unknown"
|
||||
}
|
||||
if len(hostname) > 30 {
|
||||
hostname = hostname[0:30]
|
||||
}
|
||||
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
|
||||
if apiURL == "" {
|
||||
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
|
||||
return
|
||||
}
|
||||
if apiToken == "" {
|
||||
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
|
||||
return
|
||||
}
|
||||
if youtubeAPIKey == "" {
|
||||
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
|
||||
return
|
||||
}
|
||||
if awsS3ID == "" {
|
||||
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
|
||||
return
|
||||
}
|
||||
if awsS3Secret == "" {
|
||||
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
|
||||
return
|
||||
}
|
||||
if awsS3Region == "" {
|
||||
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
|
||||
return
|
||||
}
|
||||
if awsS3Bucket == "" {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
ytUtils.SendErrorToSlack(errors.FullTrace(err))
|
||||
}
|
||||
ytUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
cmd.Execute()
|
||||
}
|
||||
+9
-3
@@ -58,9 +58,12 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
}
|
||||
|
||||
var lastChannelProcessed string
|
||||
var secondLastChannelProcessed string
|
||||
syncCount := 0
|
||||
var (
|
||||
lastChannelProcessed string
|
||||
secondLastChannelProcessed string
|
||||
syncCount int
|
||||
)
|
||||
|
||||
for {
|
||||
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
|
||||
err := s.checkUsedSpace()
|
||||
@@ -108,10 +111,12 @@ func (s *SyncManager) Start() error {
|
||||
log.Infof("Drained the \"%s\" queue", q)
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.channelsToSync) == 0 {
|
||||
log.Infoln("No channels to sync. Pausing 5 minutes!")
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
|
||||
for _, sync := range s.channelsToSync {
|
||||
if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
|
||||
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
|
||||
@@ -174,6 +179,7 @@ func (s *SyncManager) Start() error {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+13
-56
@@ -517,9 +517,14 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
claimSize := uint64(0)
|
||||
if chainInfo.Claim.Value.GetStream().Source != nil {
|
||||
claimSize, err = chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
}
|
||||
} else {
|
||||
util.SendToSlack("[%s] video with claimID %s has no source?! panic prevented...", s.DbChannelData.ChannelId, chainInfo.ClaimID)
|
||||
}
|
||||
fixed++
|
||||
log.Debugf("updating %s in the database", videoID)
|
||||
@@ -743,55 +748,7 @@ func (s *Sync) doSync() error {
|
||||
func (s *Sync) startWorker(workerNum int) {
|
||||
var v ytapi.Video
|
||||
var more bool
|
||||
fatalErrors := []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
"NotEnoughFunds",
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Missing inputs",
|
||||
}
|
||||
errorsNoRetry := []string{
|
||||
"non 200 status code received",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
}
|
||||
walletErrors := []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
//"Missing inputs",
|
||||
}
|
||||
blockchainErrors := []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
@@ -825,14 +782,14 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
err := s.processVideo(v)
|
||||
if err != nil {
|
||||
logUtils.SendErrorToSlack("error processing video %s: %s", v.ID(), err.Error())
|
||||
shouldRetry := s.Manager.CliFlags.MaxTries > 1 && !util.SubstringInSlice(err.Error(), errorsNoRetry) && tryCount < s.Manager.CliFlags.MaxTries
|
||||
shouldRetry := s.Manager.CliFlags.MaxTries > 1 && !util.SubstringInSlice(err.Error(), shared.ErrorsNoRetry) && tryCount < s.Manager.CliFlags.MaxTries
|
||||
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
|
||||
s.grp.Stop()
|
||||
} else if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
} else if util.SubstringInSlice(err.Error(), shared.FatalErrors) {
|
||||
s.hardVideoFailure.flagFailure(err.Error())
|
||||
s.grp.Stop()
|
||||
} else if shouldRetry {
|
||||
if util.SubstringInSlice(err.Error(), blockchainErrors) {
|
||||
if util.SubstringInSlice(err.Error(), shared.BlockchainErrors) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -840,7 +797,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err))
|
||||
break
|
||||
}
|
||||
} else if util.SubstringInSlice(err.Error(), walletErrors) {
|
||||
} else if util.SubstringInSlice(err.Error(), shared.WalletErrors) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
if err != nil {
|
||||
|
||||
@@ -45,6 +45,63 @@ func (p *PublishAddress) UnmarshalJSON(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var FatalErrors = []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
"NotEnoughFunds",
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Missing inputs",
|
||||
}
|
||||
var ErrorsNoRetry = []string{
|
||||
"Requested format is not available",
|
||||
"non 200 status code received",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
"Premieres in",
|
||||
"Private video",
|
||||
"This live event will begin in",
|
||||
"This video has been removed by the uploader",
|
||||
"Premiere will begin shortly",
|
||||
"cannot unmarshal number 0.0",
|
||||
"default youtube thumbnail found",
|
||||
}
|
||||
var WalletErrors = []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
//"Missing inputs",
|
||||
}
|
||||
var BlockchainErrors = []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}
|
||||
var NeverRetryFailures = []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
@@ -57,6 +114,9 @@ var NeverRetryFailures = []string{
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"uploader has not made this video available in your country",
|
||||
"This video has been removed by the uploader",
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
|
||||
+152
-94
@@ -13,6 +13,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/abadojack/whatlanggo"
|
||||
@@ -215,9 +216,7 @@ func (v *YoutubeVideo) download() error {
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.IsLive != nil {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -236,14 +235,14 @@ func (v *YoutubeVideo) download() error {
|
||||
"1080",
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
"360",
|
||||
}
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
|
||||
if dur.Hours() > 1 { //for videos longer than 1 hour only sync up to 720p
|
||||
qualities = []string{
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
"360",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,6 +274,10 @@ func (v *YoutubeVideo) download() error {
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"--extractor-args",
|
||||
"youtube:player_client=android",
|
||||
//"--concurrent-fragments",
|
||||
//"2",
|
||||
"--load-info-json",
|
||||
metadataPath,
|
||||
}
|
||||
@@ -318,11 +321,15 @@ func (v *YoutubeVideo) download() error {
|
||||
sourceAddress,
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
|
||||
)
|
||||
|
||||
//speedThrottleRetries := 3
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
//if speedThrottleRetries > 0 {
|
||||
// speedThrottleRetries--
|
||||
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
|
||||
//}
|
||||
cmd := exec.Command("yt-dlp", argsWithFilters...)
|
||||
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
@@ -338,91 +345,14 @@ func (v *YoutubeVideo) download() error {
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
dlStopGrp := stop.New()
|
||||
|
||||
ticker := time.NewTicker(400 * time.Millisecond)
|
||||
done := make(chan bool, 1)
|
||||
v.progressBarWg.Add(1)
|
||||
go func() {
|
||||
defer v.progressBarWg.Done()
|
||||
//get size of the video before downloading
|
||||
cmd := exec.Command("yt-dlp", append(argsWithFilters, "-s")...)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
go v.trackProgressBar(argsWithFilters, ticker, metadata, dlStopGrp, sourceAddress)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
output := string(outLog)
|
||||
parts := strings.Split(output, ": ")
|
||||
if len(parts) != 3 {
|
||||
log.Errorf("couldn't parse audio and video parts from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
formats := strings.Split(parts[2], "+")
|
||||
if len(formats) != 2 {
|
||||
log.Errorf("couldn't parse formats from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
log.Debugf("'%s'", output)
|
||||
videoFormat := formats[0]
|
||||
audioFormat := strings.Replace(formats[1], "\n", "", -1)
|
||||
|
||||
videoSize := 0
|
||||
audioSize := 0
|
||||
for _, f := range metadata.Formats {
|
||||
if f.FormatID == videoFormat {
|
||||
videoSize = f.Filesize
|
||||
}
|
||||
if f.FormatID == audioFormat {
|
||||
audioSize = f.Filesize
|
||||
}
|
||||
}
|
||||
log.Debugf("(%s) - videoSize: %d (%s), audiosize: %d (%s)", v.id, videoSize, videoFormat, audioSize, audioFormat)
|
||||
bar := v.progressBars.AddBar(int64(videoSize+audioSize),
|
||||
mpb.PrependDecorators(
|
||||
decor.CountersKibiByte("% .2f / % .2f "),
|
||||
// simple name decorator
|
||||
decor.Name(fmt.Sprintf("id: %s src-ip: (%s)", v.id, sourceAddress)),
|
||||
// decor.DSyncWidth bit enables column width synchronization
|
||||
decor.Percentage(decor.WCSyncSpace),
|
||||
),
|
||||
mpb.AppendDecorators(
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 90),
|
||||
decor.Name(" ] "),
|
||||
decor.EwmaSpeed(decor.UnitKiB, "% .2f ", 60),
|
||||
decor.OnComplete(
|
||||
// ETA decorator with ewma age of 60
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
|
||||
),
|
||||
),
|
||||
mpb.BarRemoveOnComplete(),
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
bar.Completed()
|
||||
bar.Abort(true)
|
||||
return
|
||||
case <-ticker.C:
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
bar.SetCurrent(size)
|
||||
if size > int64(videoSize+audioSize) {
|
||||
bar.SetTotal(size+2048, false)
|
||||
}
|
||||
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
//ticker2 := time.NewTicker(10 * time.Second)
|
||||
//v.monitorSlowDownload(ticker, dlStopGrp, sourceAddress, cmd)
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
@@ -430,7 +360,7 @@ func (v *YoutubeVideo) download() error {
|
||||
|
||||
//stop the progress bar
|
||||
ticker.Stop()
|
||||
done <- true
|
||||
dlStopGrp.Stop()
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
@@ -446,6 +376,10 @@ func (v *YoutubeVideo) download() error {
|
||||
userAgent = []string{downloader.GoogleBotUA}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
|
||||
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
|
||||
// i-- //do not lower quality when we're retrying a throttled download
|
||||
// continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
@@ -483,6 +417,126 @@ func (v *YoutubeVideo) download() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (v *YoutubeVideo) monitorSlowDownload(ticker *time.Ticker, stop *stop.Group, address string, cmd *exec.Cmd) {
|
||||
count := 0
|
||||
lastSize := int64(0)
|
||||
for {
|
||||
select {
|
||||
case <-stop.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
continue
|
||||
}
|
||||
delta := size - lastSize
|
||||
avgSpeed := delta / 10
|
||||
if avgSpeed < 200*1024 { //200 KB/s
|
||||
count++
|
||||
} else {
|
||||
count--
|
||||
}
|
||||
if count > 3 {
|
||||
err := cmd.Process.Signal(syscall.SIGKILL)
|
||||
if err != nil {
|
||||
log.Errorf("failure in killing slow download: %s", errors.Err(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.Ticker, metadata *ytMetadata, done *stop.Group, sourceAddress string) {
|
||||
v.progressBarWg.Add(1)
|
||||
go func() {
|
||||
defer v.progressBarWg.Done()
|
||||
//get size of the video before downloading
|
||||
cmd := exec.Command("yt-dlp", append(argsWithFilters, "-s")...)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
output := string(outLog)
|
||||
parts := strings.Split(output, ": ")
|
||||
if len(parts) != 3 {
|
||||
log.Errorf("couldn't parse audio and video parts from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
formats := strings.Split(parts[2], "+")
|
||||
if len(formats) != 2 {
|
||||
log.Errorf("couldn't parse formats from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
log.Debugf("'%s'", output)
|
||||
videoFormat := formats[0]
|
||||
audioFormat := strings.Replace(formats[1], "\n", "", -1)
|
||||
|
||||
videoSize := 0
|
||||
audioSize := 0
|
||||
if metadata != nil {
|
||||
for _, f := range metadata.Formats {
|
||||
if f.FormatID == videoFormat {
|
||||
videoSize = f.Filesize
|
||||
}
|
||||
if f.FormatID == audioFormat {
|
||||
audioSize = f.Filesize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("(%s) - videoSize: %d (%s), audiosize: %d (%s)", v.id, videoSize, videoFormat, audioSize, audioFormat)
|
||||
bar := v.progressBars.AddBar(int64(videoSize+audioSize),
|
||||
mpb.PrependDecorators(
|
||||
decor.CountersKibiByte("% .2f / % .2f "),
|
||||
// simple name decorator
|
||||
decor.Name(fmt.Sprintf("id: %s src-ip: (%s)", v.id, sourceAddress)),
|
||||
// decor.DSyncWidth bit enables column width synchronization
|
||||
decor.Percentage(decor.WCSyncSpace),
|
||||
),
|
||||
mpb.AppendDecorators(
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 90),
|
||||
decor.Name(" ] "),
|
||||
decor.EwmaSpeed(decor.UnitKiB, "% .2f ", 60),
|
||||
decor.OnComplete(
|
||||
// ETA decorator with ewma age of 60
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
|
||||
),
|
||||
),
|
||||
mpb.BarRemoveOnComplete(),
|
||||
)
|
||||
defer func() {
|
||||
bar.Completed()
|
||||
bar.Abort(true)
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case <-done.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
bar.SetCurrent(size)
|
||||
if size > int64(videoSize+audioSize) {
|
||||
bar.SetTotal(size+2048, false)
|
||||
}
|
||||
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
type ytMetadata struct {
|
||||
ID string `json:"id"`
|
||||
@@ -631,6 +685,9 @@ func (v *YoutubeVideo) delete(reason string) error {
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
if thumbnail.Width == 0 {
|
||||
return errors.Err("default youtube thumbnail found")
|
||||
}
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
@@ -724,6 +781,9 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
minDuration := 7 * time.Second
|
||||
|
||||
if v.youtubeInfo.IsLive == true {
|
||||
return nil, errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
@@ -742,7 +802,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
break
|
||||
}
|
||||
|
||||
//log.Debugln("Downloaded " + v.id)
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
|
||||
@@ -838,9 +897,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
return nil, errors.Prefix("the video must be republished as we can't get the right size and it doesn't exist on youtube anymore", err)
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
|
||||
+4
-6
@@ -117,13 +117,11 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
|
||||
var bestWidth *ytdl.Thumbnail
|
||||
var bestWidth ytdl.Thumbnail
|
||||
for _, thumbnail := range thumbnails {
|
||||
if bestWidth == nil {
|
||||
bestWidth = &thumbnail
|
||||
} else if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = &thumbnail
|
||||
if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = thumbnail
|
||||
}
|
||||
}
|
||||
return bestWidth
|
||||
return &bestWidth
|
||||
}
|
||||
Reference in new issue
Block a user