Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e554bbfe18 | ||
|
|
6944e17f43 | ||
|
|
a224fe44c2 | ||
|
|
fa9dc35123 | ||
|
|
3a0882230a | ||
|
|
fbd683e094 | ||
|
|
2f15c920d4 | ||
|
|
01f6448e72 | ||
|
|
8fb1e2ead0 | ||
|
|
3b84db382c | ||
|
|
4fe6840a4e | ||
|
|
69e6fb51d1 | ||
|
|
f17110ab7f | ||
|
|
768743a200 | ||
|
|
7c652b22a1 | ||
|
|
a0fb4e579e | ||
|
|
519e1e4648 | ||
|
|
087f20c133 |
No files matched your search
+1
-1
@@ -23,7 +23,7 @@ addons:
|
||||
- python3-pip
|
||||
|
||||
before_script:
|
||||
- sudo pip3 install -U youtube-dl
|
||||
- sudo pip3 install -U yt-dlp
|
||||
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
|
||||
|
||||
env:
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package cmd
|
||||
|
||||
import "github.com/lbryio/ytsync/v5/local"
|
||||
|
||||
func init() {
|
||||
local.AddCommand(rootCmd)
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
Run: ytSync,
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
rootCmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
rootCmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
rootCmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
rootCmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
|
||||
rootCmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
rootCmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
rootCmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
rootCmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
rootCmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
|
||||
rootCmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
rootCmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func ytSync(cmd *cobra.Command, args []string) {
|
||||
var hostname string
|
||||
slackToken := os.Getenv("SLACK_TOKEN")
|
||||
if slackToken == "" {
|
||||
log.Error("A slack token was not present in env vars! Slack messages disabled!")
|
||||
} else {
|
||||
var err error
|
||||
hostname, err = os.Hostname()
|
||||
if err != nil {
|
||||
log.Error("could not detect system hostname")
|
||||
hostname = "ytsync-unknown"
|
||||
}
|
||||
if len(hostname) > 30 {
|
||||
hostname = hostname[0:30]
|
||||
}
|
||||
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
|
||||
if apiURL == "" {
|
||||
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
|
||||
return
|
||||
}
|
||||
if apiToken == "" {
|
||||
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
|
||||
return
|
||||
}
|
||||
if youtubeAPIKey == "" {
|
||||
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
|
||||
return
|
||||
}
|
||||
if awsS3ID == "" {
|
||||
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
|
||||
return
|
||||
}
|
||||
if awsS3Secret == "" {
|
||||
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
|
||||
return
|
||||
}
|
||||
if awsS3Region == "" {
|
||||
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
|
||||
return
|
||||
}
|
||||
if awsS3Bucket == "" {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
)
|
||||
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
ytUtils.SendErrorToSlack(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
ytUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
}
|
||||
+42
-18
@@ -8,7 +8,9 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -16,6 +18,8 @@ import (
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
util2 "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
@@ -25,8 +29,8 @@ import (
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool, true)
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt", "--playlist-end", fmt.Sprintf("%d", maxVideos)}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -44,13 +48,31 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
|
||||
results, err := run(videoID, args, stopChan, pool, false)
|
||||
args := []string{
|
||||
"--skip-download",
|
||||
"--write-info-json",
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", videoID),
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"-o",
|
||||
path.Join(util2.GetVideoMetadataDir(), videoID),
|
||||
}
|
||||
_, err := run(videoID, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
f, err := os.Open(path.Join(util2.GetVideoMetadataDir(), videoID+".info.json"))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
// defer the closing of our jsonFile so that we can parse it later on
|
||||
defer f.Close()
|
||||
// read our opened jsonFile as a byte array.
|
||||
byteValue, _ := ioutil.ReadAll(f)
|
||||
|
||||
var video *ytdl.YtdlVideo
|
||||
err = json.Unmarshal([]byte(results[0]), &video)
|
||||
err = json.Unmarshal(byteValue, &video)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -126,7 +148,7 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
req.Header.Set("User-Agent", ChromeUA)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
@@ -196,7 +218,7 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
req.Header.Set("User-Agent", ChromeUA)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
@@ -251,16 +273,18 @@ func getClient(ip *net.TCPAddr) *http.Client {
|
||||
}
|
||||
|
||||
const (
|
||||
googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
|
||||
GoogleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
|
||||
maxAttempts = 3
|
||||
extractionError = "YouTube said: Unable to extract video data"
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
videoPremiereError = "Premieres in"
|
||||
liveEventError = "This live event will begin in"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool, dlc bool) ([]string, error) {
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var useragent []string
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
@@ -270,10 +294,7 @@ func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool,
|
||||
}
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
binary := "youtube-dl"
|
||||
if dlc {
|
||||
binary = "youtube-dlc"
|
||||
}
|
||||
binary := "yt-dlp"
|
||||
cmd := exec.Command(binary, argsForCommand...)
|
||||
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
@@ -283,6 +304,9 @@ func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool,
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if util.SubstringInSlice(err.Error(), shared.ErrorsNoRetry) {
|
||||
break
|
||||
}
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
@@ -299,13 +323,13 @@ func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool,
|
||||
|
||||
func nextUA(current []string) []string {
|
||||
if len(current) == 0 {
|
||||
return []string{"--user-agent", googleBotUA}
|
||||
return []string{"--user-agent", GoogleBotUA}
|
||||
}
|
||||
return []string{"--user-agent", chromeUA}
|
||||
return []string{"--user-agent", ChromeUA}
|
||||
}
|
||||
|
||||
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
logrus.Infof("running youtube-dl(c) cmd: %s", strings.Join(cmd.Args, " "))
|
||||
logrus.Infof("running yt-dlp cmd: %s", strings.Join(cmd.Args, " "))
|
||||
var err error
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
@@ -341,7 +365,7 @@ func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
return nil, errors.Err("interrupted by user")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("youtube-dl(c) "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
|
||||
+32
-29
@@ -63,10 +63,10 @@ type YtdlVideo struct {
|
||||
Width int `json:"width"`
|
||||
EndTime interface{} `json:"end_time"`
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
//Formats []Format `json:"formats"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Formats []Format `json:"formats"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
}
|
||||
|
||||
type RequestedFormat struct {
|
||||
@@ -103,31 +103,34 @@ type RequestedFormat struct {
|
||||
}
|
||||
|
||||
type Format struct {
|
||||
Asr int `json:"asr"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
Protocol string `json:"protocol"`
|
||||
Format string `json:"format"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Height interface{} `json:"height"`
|
||||
ManifestURL string `json:"manifest_url,omitempty"`
|
||||
FormatID string `json:"format_id"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Language interface{} `json:"language,omitempty"`
|
||||
HTTPHeaders HTTPHeaders `json:"http_headers"`
|
||||
URL string `json:"url"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Abr int `json:"abr,omitempty"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
FragmentBaseURL string `json:"fragment_base_url,omitempty"`
|
||||
Filesize interface{} `json:"filesize"`
|
||||
Fps float64 `json:"fps"`
|
||||
Fragments []struct {
|
||||
Path string `json:"path"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
} `json:"fragments,omitempty"`
|
||||
Acodec string `json:"acodec"`
|
||||
PlayerURL interface{} `json:"player_url,omitempty"`
|
||||
Asr int `json:"asr"`
|
||||
Filesize int `json:"filesize"`
|
||||
FormatID string `json:"format_id"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Fps interface{} `json:"fps"`
|
||||
Height interface{} `json:"height"`
|
||||
Quality int `json:"quality"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
URL string `json:"url"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr float64 `json:"abr,omitempty"`
|
||||
DownloaderOptions struct {
|
||||
HTTPChunkSize int `json:"http_chunk_size"`
|
||||
} `json:"downloader_options,omitempty"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Format string `json:"format"`
|
||||
Protocol string `json:"protocol"`
|
||||
HTTPHeaders struct {
|
||||
UserAgent string `json:"User-Agent"`
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
Accept string `json:"Accept"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
} `json:"http_headers"`
|
||||
Vbr float64 `json:"vbr,omitempty"`
|
||||
}
|
||||
|
||||
type Thumbnail struct {
|
||||
|
||||
+23
-1
@@ -31,12 +31,14 @@ services:
|
||||
- BANDWIDTH_LIMIT=80000000000
|
||||
- SESSION_TIMEOUT=10000000000000000000000000
|
||||
- TCP_PORT=50001
|
||||
- ELASTIC_HOST=es01
|
||||
ports:
|
||||
- "15300:50001"
|
||||
expose:
|
||||
- "50001"
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- es01
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 90000
|
||||
@@ -44,10 +46,30 @@ services:
|
||||
#command: lbry.wallet.server.coin.LBC
|
||||
command: lbry.wallet.server.coin.LBCRegTest
|
||||
#############
|
||||
## elasticsearch ##
|
||||
#############
|
||||
es01:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||
container_name: es01
|
||||
environment:
|
||||
- node.name=es01
|
||||
- discovery.type=single-node
|
||||
- indices.query.bool.max_clause_count=8196
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
ports:
|
||||
- "9200:9200"
|
||||
expose:
|
||||
- "9200"
|
||||
#############
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.90.1
|
||||
image: lbry/lbrynet:v0.99.0
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
|
||||
+2
-1
@@ -95,4 +95,5 @@ if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStat
|
||||
exit 1;
|
||||
else
|
||||
echo "SUCCESSSSSSSSSSSSS!"
|
||||
fi;
|
||||
fi;
|
||||
docker-compose down
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.90.1
|
||||
image: lbry/lbrynet:v0.99.0
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -20,7 +20,7 @@ require (
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
@@ -32,6 +32,7 @@ require (
|
||||
github.com/spf13/cobra v0.0.5
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/vbauerster/mpb/v7 v7.0.2
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
gopkg.in/ini.v1 v1.60.2 // indirect
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
|
||||
|
||||
@@ -26,8 +26,12 @@ github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+q
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
|
||||
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
@@ -279,8 +283,9 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b h1:/UnviTRIEwmkdLego+OI3Q75kKcztLuqPuJBYtcvz7w=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b/go.mod h1:K7IAMJMjDisynAcDkDdZHbZeWth6sTLl66xh+uJqtGg=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
@@ -311,6 +316,8 @@ github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJG
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
@@ -391,6 +398,8 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
|
||||
@@ -483,6 +492,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/vbauerster/mpb/v7 v7.0.2 h1:eN6AD/ytv1nqCO7Dm8MO0/pGMKmMyH/WMnTJhAUuc/w=
|
||||
github.com/vbauerster/mpb/v7 v7.0.2/go.mod h1:Mnq3gESXJ9eQhccbGZDggJ1faTCrmaA4iN57fUloRGE=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
|
||||
@@ -616,8 +627,9 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg=
|
||||
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
|
||||
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package local
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func AddCommand(rootCmd *cobra.Command) {
|
||||
cmd := &cobra.Command{
|
||||
Use: "local",
|
||||
Short: "run a personal ytsync",
|
||||
Run: localCmd,
|
||||
}
|
||||
//cmd.Flags().StringVar(&cache, "cache", "", "path to cache")
|
||||
rootCmd.AddCommand(cmd)
|
||||
|
||||
}
|
||||
|
||||
func localCmd(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("local")
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
# Running ytsync locally
|
||||
|
||||
## Requirements
|
||||
|
||||
- LBRY SDK (what do we actually need this for?)
|
||||
- youtube-dl
|
||||
- enough space to cache stuff
|
||||
|
||||
|
||||
## Process
|
||||
|
||||
### Ensuring requirements are met
|
||||
|
||||
- claim channel if there isn't one yet
|
||||
- or easier, just error if no channel
|
||||
- enough lbc in wallet?
|
||||
|
||||
### Options to figure out what's already synced
|
||||
|
||||
- simplest: assume nothing is synced yet
|
||||
- assume everything before some video is synced
|
||||
- get/put sync info from Odysee by proving you have private key for channel
|
||||
- tag videos as having been synced from youtube so we can ensure accuracy
|
||||
- hardest: scan channel and try to match up which videos are not synced yet
|
||||
|
||||
### Central DB
|
||||
|
||||
- prove you have a channel's private key to get info about that channel
|
||||
- proper queue instead of sleeping for N minutes between syncs
|
||||
|
||||
|
||||
|
||||
### Syncing a single video
|
||||
|
||||
- downloading it
|
||||
- thumbnails
|
||||
- metadata
|
||||
- having enough LBC for publish(es)
|
||||
- automated error handling
|
||||
- getting a human involved for errors that can't be handled automatically
|
||||
- reflecting
|
||||
|
||||
### Continuous Sync
|
||||
|
||||
- running in background
|
||||
- storing local state
|
||||
- interactions with our central ytsync db
|
||||
- dealing with yt throttling
|
||||
|
||||
|
||||
### Debugging
|
||||
|
||||
- dry-running the whole thing
|
||||
@@ -1,172 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/cmd"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var Version string
|
||||
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
go func() {
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
}()
|
||||
cmd := &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
Run: ytSync,
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
|
||||
cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func ytSync(cmd *cobra.Command, args []string) {
|
||||
var hostname string
|
||||
slackToken := os.Getenv("SLACK_TOKEN")
|
||||
if slackToken == "" {
|
||||
log.Error("A slack token was not present in env vars! Slack messages disabled!")
|
||||
} else {
|
||||
var err error
|
||||
hostname, err = os.Hostname()
|
||||
if err != nil {
|
||||
log.Error("could not detect system hostname")
|
||||
hostname = "ytsync-unknown"
|
||||
}
|
||||
if len(hostname) > 30 {
|
||||
hostname = hostname[0:30]
|
||||
}
|
||||
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
|
||||
if apiURL == "" {
|
||||
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
|
||||
return
|
||||
}
|
||||
if apiToken == "" {
|
||||
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
|
||||
return
|
||||
}
|
||||
if youtubeAPIKey == "" {
|
||||
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
|
||||
return
|
||||
}
|
||||
if awsS3ID == "" {
|
||||
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
|
||||
return
|
||||
}
|
||||
if awsS3Secret == "" {
|
||||
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
|
||||
return
|
||||
}
|
||||
if awsS3Region == "" {
|
||||
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
|
||||
return
|
||||
}
|
||||
if awsS3Bucket == "" {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
ytUtils.SendErrorToSlack(errors.FullTrace(err))
|
||||
}
|
||||
ytUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
cmd.Execute()
|
||||
}
|
||||
+13
-3
@@ -58,9 +58,12 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
}
|
||||
|
||||
var lastChannelProcessed string
|
||||
var secondLastChannelProcessed string
|
||||
syncCount := 0
|
||||
var (
|
||||
lastChannelProcessed string
|
||||
secondLastChannelProcessed string
|
||||
syncCount int
|
||||
)
|
||||
|
||||
for {
|
||||
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
|
||||
err := s.checkUsedSpace()
|
||||
@@ -108,10 +111,12 @@ func (s *SyncManager) Start() error {
|
||||
log.Infof("Drained the \"%s\" queue", q)
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.channelsToSync) == 0 {
|
||||
log.Infoln("No channels to sync. Pausing 5 minutes!")
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
|
||||
for _, sync := range s.channelsToSync {
|
||||
if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
|
||||
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
|
||||
@@ -153,6 +158,10 @@ func (s *SyncManager) Start() error {
|
||||
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
err = logUtils.CleanupMetadata()
|
||||
if err != nil {
|
||||
log.Errorf("something went wrong while trying to clear out the video metadata directory: %s", errors.FullTrace(err))
|
||||
}
|
||||
err = blobs_reflector.ReflectAndClean()
|
||||
if err != nil {
|
||||
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
|
||||
@@ -170,6 +179,7 @@ func (s *SyncManager) Start() error {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+24
-58
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
@@ -54,6 +55,9 @@ type Sync struct {
|
||||
queue chan ytapi.Video
|
||||
defaultAccountID string
|
||||
hardVideoFailure hardVideoFailure
|
||||
|
||||
progressBarWg *sync.WaitGroup
|
||||
progressBar *mpb.Progress
|
||||
}
|
||||
|
||||
type hardVideoFailure struct {
|
||||
@@ -128,7 +132,6 @@ func (s *Sync) FullCycle() (e error) {
|
||||
log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)")
|
||||
s.grp.Stop()
|
||||
time.Sleep(5 * time.Second)
|
||||
debug.PrintStack() // so we can figure out what's not stopping
|
||||
}()
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
@@ -176,7 +179,12 @@ func (s *Sync) FullCycle() (e error) {
|
||||
return err
|
||||
}
|
||||
|
||||
s.progressBarWg = &sync.WaitGroup{}
|
||||
s.progressBar = mpb.New(mpb.WithWaitGroup(s.progressBarWg))
|
||||
|
||||
err = s.doSync()
|
||||
// Waiting for passed &wg and for all bars to complete and flush
|
||||
s.progressBar.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -509,9 +517,14 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
}
|
||||
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
|
||||
claimSize, err := chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
claimSize := uint64(0)
|
||||
if chainInfo.Claim.Value.GetStream().Source != nil {
|
||||
claimSize, err = chainInfo.Claim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
claimSize = 0
|
||||
}
|
||||
} else {
|
||||
util.SendToSlack("[%s] video with claimID %s has no source?! panic prevented...", s.DbChannelData.ChannelId, chainInfo.ClaimID)
|
||||
}
|
||||
fixed++
|
||||
log.Debugf("updating %s in the database", videoID)
|
||||
@@ -735,55 +748,7 @@ func (s *Sync) doSync() error {
|
||||
func (s *Sync) startWorker(workerNum int) {
|
||||
var v ytapi.Video
|
||||
var more bool
|
||||
fatalErrors := []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
"NotEnoughFunds",
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Missing inputs",
|
||||
}
|
||||
errorsNoRetry := []string{
|
||||
"non 200 status code received",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
}
|
||||
walletErrors := []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
//"Missing inputs",
|
||||
}
|
||||
blockchainErrors := []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
@@ -813,17 +778,18 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
default:
|
||||
}
|
||||
tryCount++
|
||||
|
||||
err := s.processVideo(v)
|
||||
if err != nil {
|
||||
logUtils.SendErrorToSlack("error processing video %s: %s", v.ID(), err.Error())
|
||||
shouldRetry := s.Manager.CliFlags.MaxTries > 1 && !util.SubstringInSlice(err.Error(), errorsNoRetry) && tryCount < s.Manager.CliFlags.MaxTries
|
||||
shouldRetry := s.Manager.CliFlags.MaxTries > 1 && !util.SubstringInSlice(err.Error(), shared.ErrorsNoRetry) && tryCount < s.Manager.CliFlags.MaxTries
|
||||
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
|
||||
s.grp.Stop()
|
||||
} else if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
} else if util.SubstringInSlice(err.Error(), shared.FatalErrors) {
|
||||
s.hardVideoFailure.flagFailure(err.Error())
|
||||
s.grp.Stop()
|
||||
} else if shouldRetry {
|
||||
if util.SubstringInSlice(err.Error(), blockchainErrors) {
|
||||
if util.SubstringInSlice(err.Error(), shared.BlockchainErrors) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -831,7 +797,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err))
|
||||
break
|
||||
}
|
||||
} else if util.SubstringInSlice(err.Error(), walletErrors) {
|
||||
} else if util.SubstringInSlice(err.Error(), shared.WalletErrors) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
if err != nil {
|
||||
@@ -986,7 +952,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
|
||||
DefaultAccount: da,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux, s.progressBarWg, s.progressBar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -45,6 +45,63 @@ func (p *PublishAddress) UnmarshalJSON(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var FatalErrors = []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
"NotEnoughFunds",
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Missing inputs",
|
||||
}
|
||||
var ErrorsNoRetry = []string{
|
||||
"Requested format is not available",
|
||||
"non 200 status code received",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
"Premieres in",
|
||||
"Private video",
|
||||
"This live event will begin in",
|
||||
"This video has been removed by the uploader",
|
||||
"Premiere will begin shortly",
|
||||
"cannot unmarshal number 0.0",
|
||||
"default youtube thumbnail found",
|
||||
}
|
||||
var WalletErrors = []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
//"Missing inputs",
|
||||
}
|
||||
var BlockchainErrors = []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}
|
||||
var NeverRetryFailures = []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
@@ -57,6 +114,9 @@ var NeverRetryFailures = []string{
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"uploader has not made this video available in your country",
|
||||
"This video has been removed by the uploader",
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
|
||||
+319
-21
@@ -2,20 +2,26 @@ package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/abadojack/whatlanggo"
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
"github.com/vbauerster/mpb/v7/decor"
|
||||
"gopkg.in/vansante/go-ffprobe.v2"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
@@ -56,6 +62,8 @@ type YoutubeVideo struct {
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
pool *ip_manager.IPPool
|
||||
progressBars *mpb.Progress
|
||||
progressBarWg *sync.WaitGroup
|
||||
}
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
@@ -183,15 +191,32 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
}
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
func checkCookiesIntegrity() error {
|
||||
fi, err := os.Stat("cookies.txt")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if fi.Size() == 0 {
|
||||
log.Errorf("cookies were cleared out. Attempting a restore from cookies-backup.txt")
|
||||
input, err := ioutil.ReadFile("cookies-backup.txt")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile("cookies.txt", input, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.IsLive != nil {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -210,32 +235,54 @@ func (v *YoutubeVideo) download() error {
|
||||
"1080",
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
"360",
|
||||
}
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
|
||||
if dur.Hours() > 1 { //for videos longer than 1 hour only sync up to 720p
|
||||
qualities = []string{
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
"360",
|
||||
}
|
||||
}
|
||||
|
||||
metadataPath := path.Join(logUtils.GetVideoMetadataDir(), v.id+".info.json")
|
||||
_, err = os.Stat(metadataPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return errors.Err("metadata information for video %s is missing! Why?", v.id)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
metadata, err := parseVideoMetadata(metadataPath)
|
||||
|
||||
err = checkCookiesIntegrity()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--rm-cache-dir",
|
||||
"--postprocessor-args",
|
||||
"-movflags faststart",
|
||||
"ffmpeg:-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"--extractor-args",
|
||||
"youtube:player_client=android",
|
||||
//"--concurrent-fragments",
|
||||
//"2",
|
||||
"--load-info-json",
|
||||
metadataPath,
|
||||
}
|
||||
userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"}
|
||||
|
||||
userAgent := []string{"--user-agent", downloader.ChromeUA}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--max-filesize",
|
||||
@@ -272,15 +319,19 @@ func (v *YoutubeVideo) download() error {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
|
||||
)
|
||||
|
||||
//speedThrottleRetries := 3
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
//if speedThrottleRetries > 0 {
|
||||
// speedThrottleRetries--
|
||||
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
|
||||
//}
|
||||
cmd := exec.Command("yt-dlp", argsWithFilters...)
|
||||
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
@@ -295,9 +346,22 @@ func (v *YoutubeVideo) download() error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
dlStopGrp := stop.New()
|
||||
|
||||
ticker := time.NewTicker(400 * time.Millisecond)
|
||||
go v.trackProgressBar(argsWithFilters, ticker, metadata, dlStopGrp, sourceAddress)
|
||||
|
||||
//ticker2 := time.NewTicker(10 * time.Second)
|
||||
//v.monitorSlowDownload(ticker, dlStopGrp, sourceAddress, cmd)
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
|
||||
//stop the progress bar
|
||||
ticker.Stop()
|
||||
dlStopGrp.Stop()
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
@@ -309,9 +373,13 @@ func (v *YoutubeVideo) download() error {
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
|
||||
i-- //do not lower quality when trying a different user agent
|
||||
userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
|
||||
userAgent = []string{downloader.GoogleBotUA}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
|
||||
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
|
||||
// i-- //do not lower quality when we're retrying a throttled download
|
||||
// continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
@@ -349,10 +417,234 @@ func (v *YoutubeVideo) download() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (v *YoutubeVideo) monitorSlowDownload(ticker *time.Ticker, stop *stop.Group, address string, cmd *exec.Cmd) {
|
||||
count := 0
|
||||
lastSize := int64(0)
|
||||
for {
|
||||
select {
|
||||
case <-stop.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
continue
|
||||
}
|
||||
delta := size - lastSize
|
||||
avgSpeed := delta / 10
|
||||
if avgSpeed < 200*1024 { //200 KB/s
|
||||
count++
|
||||
} else {
|
||||
count--
|
||||
}
|
||||
if count > 3 {
|
||||
err := cmd.Process.Signal(syscall.SIGKILL)
|
||||
if err != nil {
|
||||
log.Errorf("failure in killing slow download: %s", errors.Err(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.Ticker, metadata *ytMetadata, done *stop.Group, sourceAddress string) {
|
||||
v.progressBarWg.Add(1)
|
||||
go func() {
|
||||
defer v.progressBarWg.Done()
|
||||
//get size of the video before downloading
|
||||
cmd := exec.Command("yt-dlp", append(argsWithFilters, "-s")...)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
output := string(outLog)
|
||||
parts := strings.Split(output, ": ")
|
||||
if len(parts) != 3 {
|
||||
log.Errorf("couldn't parse audio and video parts from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
formats := strings.Split(parts[2], "+")
|
||||
if len(formats) != 2 {
|
||||
log.Errorf("couldn't parse formats from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
log.Debugf("'%s'", output)
|
||||
videoFormat := formats[0]
|
||||
audioFormat := strings.Replace(formats[1], "\n", "", -1)
|
||||
|
||||
videoSize := 0
|
||||
audioSize := 0
|
||||
if metadata != nil {
|
||||
for _, f := range metadata.Formats {
|
||||
if f.FormatID == videoFormat {
|
||||
videoSize = f.Filesize
|
||||
}
|
||||
if f.FormatID == audioFormat {
|
||||
audioSize = f.Filesize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("(%s) - videoSize: %d (%s), audiosize: %d (%s)", v.id, videoSize, videoFormat, audioSize, audioFormat)
|
||||
bar := v.progressBars.AddBar(int64(videoSize+audioSize),
|
||||
mpb.PrependDecorators(
|
||||
decor.CountersKibiByte("% .2f / % .2f "),
|
||||
// simple name decorator
|
||||
decor.Name(fmt.Sprintf("id: %s src-ip: (%s)", v.id, sourceAddress)),
|
||||
// decor.DSyncWidth bit enables column width synchronization
|
||||
decor.Percentage(decor.WCSyncSpace),
|
||||
),
|
||||
mpb.AppendDecorators(
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 90),
|
||||
decor.Name(" ] "),
|
||||
decor.EwmaSpeed(decor.UnitKiB, "% .2f ", 60),
|
||||
decor.OnComplete(
|
||||
// ETA decorator with ewma age of 60
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
|
||||
),
|
||||
),
|
||||
mpb.BarRemoveOnComplete(),
|
||||
)
|
||||
defer func() {
|
||||
bar.Completed()
|
||||
bar.Abort(true)
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case <-done.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
bar.SetCurrent(size)
|
||||
if size > int64(videoSize+audioSize) {
|
||||
bar.SetTotal(size+2048, false)
|
||||
}
|
||||
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
type ytMetadata struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Formats []struct {
|
||||
Asr int `json:"asr"`
|
||||
Filesize int `json:"filesize"`
|
||||
FormatID string `json:"format_id"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Fps interface{} `json:"fps"`
|
||||
Height interface{} `json:"height"`
|
||||
Quality int `json:"quality"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
URL string `json:"url"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr float64 `json:"abr,omitempty"`
|
||||
DownloaderOptions struct {
|
||||
HTTPChunkSize int `json:"http_chunk_size"`
|
||||
} `json:"downloader_options,omitempty"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Format string `json:"format"`
|
||||
Protocol string `json:"protocol"`
|
||||
HTTPHeaders struct {
|
||||
UserAgent string `json:"User-Agent"`
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
Accept string `json:"Accept"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
} `json:"http_headers"`
|
||||
Vbr float64 `json:"vbr,omitempty"`
|
||||
} `json:"formats"`
|
||||
Thumbnails []struct {
|
||||
Height int `json:"height"`
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Resolution string `json:"resolution"`
|
||||
ID string `json:"id"`
|
||||
} `json:"thumbnails"`
|
||||
Description string `json:"description"`
|
||||
UploadDate string `json:"upload_date"`
|
||||
Uploader string `json:"uploader"`
|
||||
UploaderID string `json:"uploader_id"`
|
||||
UploaderURL string `json:"uploader_url"`
|
||||
ChannelID string `json:"channel_id"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Duration int `json:"duration"`
|
||||
ViewCount int `json:"view_count"`
|
||||
AverageRating float64 `json:"average_rating"`
|
||||
AgeLimit int `json:"age_limit"`
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
Categories []string `json:"categories"`
|
||||
Tags []interface{} `json:"tags"`
|
||||
IsLive interface{} `json:"is_live"`
|
||||
LikeCount int `json:"like_count"`
|
||||
DislikeCount int `json:"dislike_count"`
|
||||
Channel string `json:"channel"`
|
||||
Extractor string `json:"extractor"`
|
||||
WebpageURLBasename string `json:"webpage_url_basename"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
Playlist interface{} `json:"playlist"`
|
||||
PlaylistIndex interface{} `json:"playlist_index"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
DisplayID string `json:"display_id"`
|
||||
Format string `json:"format"`
|
||||
FormatID string `json:"format_id"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Fps int `json:"fps"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Vbr float64 `json:"vbr"`
|
||||
StretchedRatio interface{} `json:"stretched_ratio"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr float64 `json:"abr"`
|
||||
Ext string `json:"ext"`
|
||||
Fulltitle string `json:"fulltitle"`
|
||||
Filename string `json:"_filename"`
|
||||
}
|
||||
|
||||
func parseVideoMetadata(metadataPath string) (*ytMetadata, error) {
|
||||
f, err := os.Open(metadataPath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
// defer the closing of our jsonFile so that we can parse it later on
|
||||
defer f.Close()
|
||||
// read our opened jsonFile as a byte array.
|
||||
byteValue, _ := ioutil.ReadAll(f)
|
||||
|
||||
// we initialize our Users array
|
||||
var m ytMetadata
|
||||
|
||||
// we unmarshal our byteArray which contains our
|
||||
// jsonFile's content into 'users' which we defined above
|
||||
err = json.Unmarshal(byteValue, &m)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) videoDir() string {
|
||||
return v.dir + "/" + v.id
|
||||
return path.Join(v.dir, v.id)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
files, err := ioutil.ReadDir(v.videoDir())
|
||||
log.Infoln(v.videoDir())
|
||||
@@ -367,7 +659,7 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
|
||||
return v.videoDir() + "/" + f.Name(), nil
|
||||
return path.Join(v.videoDir(), f.Name()), nil
|
||||
}
|
||||
}
|
||||
return "", errors.Err("could not find any downloaded videos")
|
||||
@@ -393,6 +685,9 @@ func (v *YoutubeVideo) delete(reason string) error {
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
if thumbnail.Width == 0 {
|
||||
return errors.Err("default youtube thumbnail found")
|
||||
}
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
@@ -466,11 +761,13 @@ type SyncParams struct {
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex, pbWg *sync.WaitGroup, pb *mpb.Progress) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize)
|
||||
v.maxVideoLength = params.MaxVideoLength
|
||||
v.lbryChannelID = params.ChannelID
|
||||
v.walletLock = walletLock
|
||||
v.progressBars = pb
|
||||
v.progressBarWg = pbWg
|
||||
if reprocess && existingVideoData != nil && existingVideoData.Published {
|
||||
summary, err := v.reprocess(daemon, params, existingVideoData)
|
||||
return summary, errors.Prefix("upgrade failed", err)
|
||||
@@ -484,6 +781,9 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
minDuration := 7 * time.Second
|
||||
|
||||
if v.youtubeInfo.IsLive == true {
|
||||
return nil, errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
@@ -502,7 +802,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
|
||||
@@ -598,9 +897,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
return nil, errors.Prefix("the video must be republished as we can't get the right size and it doesn't exist on youtube anymore", err)
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
|
||||
+4
-6
@@ -117,13 +117,11 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
|
||||
var bestWidth *ytdl.Thumbnail
|
||||
var bestWidth ytdl.Thumbnail
|
||||
for _, thumbnail := range thumbnails {
|
||||
if bestWidth == nil {
|
||||
bestWidth = &thumbnail
|
||||
} else if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = &thumbnail
|
||||
if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = thumbnail
|
||||
}
|
||||
}
|
||||
return bestWidth
|
||||
return &bestWidth
|
||||
}
|
||||
@@ -261,6 +261,27 @@ func CleanupLbrynet() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var metadataDirInitialized = false
|
||||
|
||||
func GetVideoMetadataDir() string {
|
||||
dir := "./videos_metadata"
|
||||
if !metadataDirInitialized {
|
||||
metadataDirInitialized = true
|
||||
_ = os.MkdirAll(dir, 0755)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func CleanupMetadata() error {
|
||||
dir := GetVideoMetadataDir()
|
||||
err := os.RemoveAll(dir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
metadataDirInitialized = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func SleepUntilQuotaReset() {
|
||||
PST, _ := time.LoadLocation("America/Los_Angeles")
|
||||
t := time.Now().In(PST)
|
||||
@@ -384,3 +405,17 @@ func GetBlockchainDirectoryName() string {
|
||||
}
|
||||
return ledger
|
||||
}
|
||||
|
||||
func DirSize(path string) (int64, error) {
|
||||
var size int64
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
}
|
||||
return err
|
||||
})
|
||||
return size, err
|
||||
}
|
||||
+4
-3
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
@@ -38,7 +39,7 @@ type Video interface {
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex, *sync.WaitGroup, *mpb.Progress) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
type byPublishedAt []Video
|
||||
@@ -129,7 +130,7 @@ func CountVideosInChannel(channelID string) (int, error) {
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
req.Header.Add("User-Agent", downloader.ChromeUA)
|
||||
req.Header.Add("Accept", "*/*")
|
||||
req.Header.Add("Host", "socialblade.com")
|
||||
|
||||
@@ -173,7 +174,7 @@ func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
req.Header.Add("User-Agent", downloader.ChromeUA)
|
||||
req.Header.Add("Accept", "*/*")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
|
||||
Reference in new issue
Block a user