Compare commits

..
8 Commits
Author SHA1 Message Date
Niko Storni 84790720ff improve error handling
retry wallet uploads on failure
2021-12-02 16:59:14 +01:00
Niko 23690731af fix bug when uploading wallet 2021-11-30 02:53:59 +01:00
Niko Storni 75628d8530 delete tars after use 2021-11-25 04:26:04 +01:00
Niko Storni 6e819b20f6 update readme 2021-11-24 18:58:38 +01:00
Niko Storni da0b6e5b79 add example config 2021-11-24 18:39:24 +01:00
Niko Storni 28791f317b update gitignore
improve error logging to slack
fix regression in dev builds
2021-11-24 18:37:04 +01:00
Niko Storni 070287716b switch from env vars to a config file
get rid of stupid stuff
simplify S3 configuration
split wallets from blockchain.db and use separate S3 store
fix bugs
2021-11-24 05:54:08 +01:00
Alex Grin 41054e77a6 Update README.md 2021-09-28 10:20:17 -04:00
27 changed files with 574 additions and 388 deletions

No files matched your search

+4
View File
@@ -4,3 +4,7 @@ e2e/supporty/supporty
.env
blobsfiles
ytsync_docker
e2e/config.json
e2e/cookies.txt
+111
View File
@@ -0,0 +1,111 @@
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 16 Dec 2019 00:13:36 +0100
Subject: [PATCH] lbry-patch
---
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
1 file changed, 35 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b913d07a6..cd66a5b01 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -10,6 +10,7 @@ import random
import re
import time
import traceback
+import subprocess
from .common import InfoExtractor, SearchInfoExtractor
from ..jsinterp import JSInterpreter
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
_GEO_BYPASS = False
+ _WGET_429_RATE_LIMIT = 8191
+ _WGET_BINARY = "wget"
+
IE_NAME = 'youtube'
_TESTS = [
{
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
""" Return a string representation of a signature """
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
+ def _rate_limit_download(self, url, video_id, note=None):
+ if note is None:
+ self.report_download_webpage(video_id)
+ elif note is not False:
+ if video_id is None:
+ self.to_screen('%s' % (note,))
+ else:
+ self.to_screen('%s: %s' % (video_id, note))
+ source_address = self._downloader.params.get('source_address')
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
+
def _extract_signature_function(self, video_id, player_url, example_sig):
id_m = re.match(
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Get video webpage
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
- video_webpage = self._download_webpage(url, video_id)
+ video_webpage = self._rate_limit_download(url, video_id)
# Attempt to extract SWF player URL
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
})
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
- video_info_webpage = self._download_webpage(
+ video_info_webpage = self._rate_limit_download(
video_info_url, video_id,
- note='Refetching age-gated info webpage',
- errnote='unable to download video info webpage')
+ note='Refetching age-gated info webpage')
video_info = compat_parse_qs(video_info_webpage)
pl_response = video_info.get('player_response', [None])[0]
player_response = extract_player_response(pl_response, video_id)
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# The general idea is to take a union of itags of both DASH manifests (for example
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
self.report_video_info_webpage_download(video_id)
- for el in ('embedded', 'detailpage', 'vevo', ''):
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
query = {
'video_id': video_id,
'ps': 'default',
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
query['el'] = el
if sts:
query['sts'] = sts
- video_info_webpage = self._download_webpage(
- '%s://www.youtube.com/get_video_info' % proto,
- video_id, note=False,
- errnote='unable to download video info webpage',
- fatal=False, query=query)
+
+ if el == '':
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
+ else:
+ base_url = 'https://youtube.com/get_video_info'
+
+ for q in query:
+ if q is None or q is "":
+ continue
+ if query[q] is None or query[q] is "":
+ continue
+
+ base_url = base_url + "?{}={}".format(q, query[q])
+
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
+
if not video_info_webpage:
continue
get_video_info = compat_parse_qs(video_info_webpage)
--
2.17.1
+18 -21
View File
@@ -8,23 +8,17 @@ With the support of said database, the tool is also able to keep all the channel
# Requirements
- lbrynet SDK https://github.com/lbryio/lbry/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
- lbrynet SDK https://github.com/lbryio/lbry-sdk/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
- a lbrycrd node running (localhost or on a remote machine) with credits in it
- internal-apis (you cannot run this one yourself)
- python3-pip
- yt-dlp (`pip3 install -U yt-dlp`)
- ffmpeg (latest)
# Setup
- make sure daemon is stopped and can be controlled through `systemctl` (find example below)
- extract the ytsync binary anywhere
- add the environment variables necessary to the tool
- export SLACK_TOKEN="a-token-to-spam-your-slack"
- export SLACK_CHANNEL="youtube-status"
- export YOUTUBE_API_KEY="youtube-api-key"
- export LBRY_WEB_API="https://lbry-api-url-here"
- export LBRY_API_TOKEN="internal-apis-token-for-ytsync-user"
- export LBRYCRD_STRING="tcp://user:password@host:5429"
- export AWS_S3_ID="THE-ID-LIES-HERE"
- export AWS_S3_SECRET="THE-SECRET-LIES-HERE"
- export AWS_S3_REGION="us-east-1"
- export AWS_S3_BUCKET="ytsync-wallets"
- create and fill `config.json` using [this example](config.json.example)
## systemd script example
`/etc/systemd/system/lbrynet.service`
@@ -55,23 +49,26 @@ Usage:
Flags:
--after int Specify from when to pull jobs [Unix time](Default: 0)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1669311891)
--channelID string If specified, only this channel will be synced.
--concurrent-jobs int how many jobs to process concurrently (default 1)
-h, --help help for ytsync
--limit int limit the amount of channels to sync
--max-length float Maximum video length to process (in hours) (default 2)
--max-length int Maximum video length to process (in hours) (default 2)
--max-size int Maximum video size to process (in MB) (default 2048)
--max-tries int Number of times to try a publish that fails (default 3)
--no-transfers Skips the transferring process of videos, channels and supports
--quick Look up only the last 50 videos from youtube
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
--run-once Whether the process should be stopped after one cycle or not
--skip-space-check Do not perform free space check on startup
--status string Specify which queue to pull from. Overrides --update
--stop-on-error If a publish fails, stop all publishing and exit
--status2 string Specify which secondary queue to pull from.
--takeover-existing-channel If channel exists and we don't own it, take over the channel
--update Update previously synced channels instead of syncing new ones
--upgrade-metadata Upgrade videos if they're on the old metadata version
--videos-limit int how many videos to process per channel (default 1000)
--videos-limit int how many videos to process per channel (leave 0 for automatic detection)
```
## Running from Source
@@ -88,17 +85,17 @@ Contributions to this project are welcome, encouraged, and compensated. For more
## Security
We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
## Contact
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.io).
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.com).
## Additional Info and Links
- [https://lbry.io](https://lbry.io) - The live LBRY website
- [Discord Chat](https://chat.lbry.io) - A chat room for the LBRYians
- [Email us](mailto:hello@lbry.io) - LBRY Support email
- [https://lbry.com](https://lbry.com) - The live LBRY website
- [Discord Chat](https://chat.lbry.com) - A chat room for the LBRYians
- [Email us](mailto:hello@lbry.com) - LBRY Support email
- [Twitter](https://twitter.com/@lbryio) - LBRY Twitter page
- [Facebook](https://www.facebook.com/lbryio/) - LBRY Facebook page
- [Reddit](https://reddit.com/r/lbry) - LBRY Reddit page
-7
View File
@@ -1,7 +0,0 @@
package cmd
import "github.com/lbryio/ytsync/v5/local"
func init() {
local.AddCommand(rootCmd)
}
-166
View File
@@ -1,166 +0,0 @@
package cmd
import (
"os"
"time"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
const defaultMaxTries = 3
var (
cliFlags shared.SyncFlags
maxVideoLength int
)
var rootCmd = &cobra.Command{
Use: "ytsync",
Short: "Publish youtube channels into LBRY network automatically.",
Run: ytSync,
Args: cobra.RangeArgs(0, 0),
}
func init() {
rootCmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
rootCmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
rootCmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
rootCmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
rootCmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
rootCmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
rootCmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
rootCmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
rootCmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
rootCmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
rootCmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
rootCmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
rootCmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
rootCmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
rootCmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
rootCmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
rootCmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
rootCmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
rootCmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
}
func Execute() {
err := rootCmd.Execute()
if err != nil {
log.Errorln(err)
os.Exit(1)
}
}
func ytSync(cmd *cobra.Command, args []string) {
var hostname string
slackToken := os.Getenv("SLACK_TOKEN")
if slackToken == "" {
log.Error("A slack token was not present in env vars! Slack messages disabled!")
} else {
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync-unknown"
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
}
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
return
}
if cliFlags.MaxTries < 1 {
log.Errorln("setting --max-tries less than 1 doesn't make sense")
return
}
if cliFlags.Limit < 0 {
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
return
}
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
apiURL := os.Getenv("LBRY_WEB_API")
apiToken := os.Getenv("LBRY_API_TOKEN")
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
awsS3ID := os.Getenv("AWS_S3_ID")
awsS3Secret := os.Getenv("AWS_S3_SECRET")
awsS3Region := os.Getenv("AWS_S3_REGION")
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
if apiURL == "" {
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
return
}
if apiToken == "" {
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
return
}
if youtubeAPIKey == "" {
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
return
}
if awsS3ID == "" {
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
return
}
if awsS3Secret == "" {
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
return
}
if awsS3Region == "" {
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
return
}
if awsS3Bucket == "" {
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
return
}
if lbrycrdDsn == "" {
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
}
blobsDir := ytUtils.GetBlobsDir()
apiConfig := &sdk.APIConfig{
YoutubeAPIKey: youtubeAPIKey,
ApiURL: apiURL,
ApiToken: apiToken,
HostName: hostname,
}
awsConfig := &shared.AwsConfigs{
AwsS3ID: awsS3ID,
AwsS3Secret: awsS3Secret,
AwsS3Region: awsS3Region,
AwsS3Bucket: awsS3Bucket,
}
sm := manager.NewSyncManager(
cliFlags,
blobsDir,
lbrycrdDsn,
awsConfig,
apiConfig,
)
err := sm.Start()
if err != nil {
ytUtils.SendErrorToSlack(errors.FullTrace(err))
}
ytUtils.SendInfoToSlack("Syncing process terminated!")
}
+35
View File
@@ -0,0 +1,35 @@
{
"slack_token": "",
"slack_channel": "ytsync-dev",
"internal_apis_endpoint": "http://localhost:15400",
"internal_apis_auth_token": "ytsyntoken",
"lbrycrd_string": "tcp://lbry:lbry@localhost:15200",
"wallet_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "ytsync-wallets",
"endpoint": ""
},
"blockchaindb_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "blockchaindbs",
"endpoint": ""
},
"thumbnails_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "thumbnails.lbry.com",
"endpoint": ""
},
"aws_thumbnails_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "thumbnails.lbry.com",
"endpoint": ""
}
}
+52
View File
@@ -0,0 +1,52 @@
package configs
import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/tkanos/gonfig"
)
type S3Configs struct {
ID string `json:"id"`
Secret string `json:"secret"`
Region string `json:"region"`
Bucket string `json:"bucket"`
Endpoint string `json:"endpoint"`
}
type Configs struct {
SlackToken string `json:"slack_token"`
SlackChannel string `json:"slack_channel"`
InternalApisEndpoint string `json:"internal_apis_endpoint"`
InternalApisAuthToken string `json:"internal_apis_auth_token"`
LbrycrdString string `json:"lbrycrd_string"`
WalletS3Config S3Configs `json:"wallet_s3_config"`
BlockchaindbS3Config S3Configs `json:"blockchaindb_s3_config"`
AWSThumbnailsS3Config S3Configs `json:"aws_thumbnails_s3_config"`
ThumbnailsS3Config S3Configs `json:"thumbnails_s3_config"`
}
var Configuration *Configs
func Init(configPath string) error {
if Configuration != nil {
return nil
}
c := Configs{}
err := gonfig.GetConf(configPath, &c)
if err != nil {
return errors.Err(err)
}
Configuration = &c
return nil
}
func (s *S3Configs) GetS3AWSConfig() *aws.Config {
return &aws.Config{
Credentials: credentials.NewStaticCredentials(s.ID, s.Secret, ""),
Region: &s.Region,
Endpoint: &s.Endpoint,
S3ForcePathStyle: aws.Bool(true),
}
}
+3
View File
@@ -36,6 +36,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
}
videoIDs := make([]string, 0, maxVideos)
for i, v := range ids {
if v == "" {
continue
}
logrus.Debugf("%d - video id %s", i, v)
if i >= maxVideos {
break
+1 -6
View File
@@ -29,12 +29,7 @@ func TestGetVideoInformation(t *testing.T) {
}
func Test_getUploadTime(t *testing.T) {
configs := sdk.APIConfig{
YoutubeAPIKey: "",
ApiURL: "https://api.lbry.com",
ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
HostName: "test",
}
configs := sdk.APIConfig{}
got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
assert.NoError(t, err)
t.Log(got)
+4 -7
View File
@@ -14,11 +14,8 @@ export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
touch -a .env && set -o allexport; source ./.env; set +o allexport
echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
# Compose settings - docker only
export SLACK_CHANNEL="ytsync-travis"
export LBRY_API_TOKEN="ytsyntoken"
export LBRY_WEB_API="http://localhost:15400"
export LBRYNET_ADDRESS="http://localhost:15100"
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200" #required for supporty
export LBRYNET_USE_DOCKER=true
export REFLECT_BLOBS=false
export CLEAN_ON_STARTUP=true
@@ -50,9 +47,9 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
done
echo "successfully started..."
channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
channelName=@Alaminemoh11"$(date +%s)"
latestVideoID="ejWF7Jjdgmc"
channelToSync="UCMn-zv1SE-2y6vyewscfFqw"
channelName=@whatever"$(date +%s)"
latestVideoID="yPJgjiMbmX0"
#Data Setup for test
./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
+1
View File
@@ -32,6 +32,7 @@ require (
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0
github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
github.com/vbauerster/mpb/v7 v7.0.2
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/ini.v1 v1.60.2 // indirect
+3
View File
@@ -111,6 +111,7 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
@@ -488,6 +489,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f h1:xDFq4NVQD34ekH5UsedBSgfxsBuPU2aZf7v4t0tH2jY=
github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f/go.mod h1:DaZPBuToMc2eezA9R9nDAnmS2RMwL7yEa5YD36ESQdI=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-22
View File
@@ -1,22 +0,0 @@
package local
import (
"fmt"
"github.com/spf13/cobra"
)
func AddCommand(rootCmd *cobra.Command) {
cmd := &cobra.Command{
Use: "local",
Short: "run a personal ytsync",
Run: localCmd,
}
//cmd.Flags().StringVar(&cache, "cache", "", "path to cache")
rootCmd.AddCommand(cmd)
}
func localCmd(cmd *cobra.Command, args []string) {
fmt.Println("local")
}
-53
View File
@@ -1,53 +0,0 @@
# Running ytsync locally
## Requirements
- LBRY SDK (what do we actually need this for?)
- youtube-dl
- enough space to cache stuff
## Process
### Ensuring requirements are met
- claim channel if there isn't one yet
- or easier, just error if no channel
- enough lbc in wallet?
### Options to figure out what's already synced
- simplest: assume nothing is synced yet
- assume everything before some video is synced
- get/put sync info from Odysee by proving you have private key for channel
- tag videos as having been synced from youtube so we can ensure accuracy
- hardest: scan channel and try to match up which videos are not synced yet
### Central DB
- prove you have a channel's private key to get info about that channel
- proper queue instead of sleeping for N minutes between syncs
### Syncing a single video
- downloading it
- thumbnails
- metadata
- having enough LBC for publish(es)
- automated error handling
- getting a human involved for errors that can't be handled automatically
- reflecting
### Continuous Sync
- running in background
- storing local state
- interactions with our central ytsync db
- dealing with yt throttling
### Debugging
- dry-running the whole thing
+131 -4
View File
@@ -1,24 +1,151 @@
package main
import (
"fmt"
"math/rand"
"net/http"
"os"
"time"
"github.com/lbryio/ytsync/v5/cmd"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var Version string
const defaultMaxTries = 3
var (
cliFlags shared.SyncFlags
maxVideoLength int
)
func main() {
rand.Seed(time.Now().UnixNano())
log.SetLevel(log.DebugLevel)
http.Handle("/metrics", promhttp.Handler())
go func() {
http.Handle("/metrics", promhttp.Handler())
log.Error(http.ListenAndServe(":2112", nil))
}()
cmd := &cobra.Command{
Use: "ytsync",
Short: "Publish youtube channels into LBRY network automatically.",
Run: ytSync,
Args: cobra.RangeArgs(0, 0),
}
cmd.Execute()
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
cmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
if err := cmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func ytSync(cmd *cobra.Command, args []string) {
err := configs.Init("./config.json")
if err != nil {
log.Fatalf("could not parse configuration file: %s", errors.FullTrace(err))
}
var hostname string
if configs.Configuration.SlackToken == "" {
log.Error("A slack token was not present in the config! Slack messages disabled!")
} else {
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync-unknown"
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, hostname)
}
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
return
}
if cliFlags.MaxTries < 1 {
log.Errorln("setting --max-tries less than 1 doesn't make sense")
return
}
if cliFlags.Limit < 0 {
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
return
}
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
if configs.Configuration.InternalApisEndpoint == "" {
log.Errorln("An Internal APIs Endpoint was not defined")
return
}
if configs.Configuration.InternalApisAuthToken == "" {
log.Errorln("An Internal APIs auth token was not defined")
return
}
if configs.Configuration.WalletS3Config.ID == "" || configs.Configuration.WalletS3Config.Region == "" || configs.Configuration.WalletS3Config.Bucket == "" || configs.Configuration.WalletS3Config.Secret == "" || configs.Configuration.WalletS3Config.Endpoint == "" {
log.Errorln("Wallet S3 configuration is incomplete")
return
}
if configs.Configuration.BlockchaindbS3Config.ID == "" || configs.Configuration.BlockchaindbS3Config.Region == "" || configs.Configuration.BlockchaindbS3Config.Bucket == "" || configs.Configuration.BlockchaindbS3Config.Secret == "" || configs.Configuration.BlockchaindbS3Config.Endpoint == "" {
log.Errorln("Blockchain DBs S3 configuration is incomplete")
return
}
if configs.Configuration.LbrycrdString == "" {
log.Infoln("Using default (local) lbrycrd instance. Set lbrycrd_string if you want to use something else")
}
blobsDir := ytUtils.GetBlobsDir()
apiConfig := &sdk.APIConfig{
ApiURL: configs.Configuration.InternalApisEndpoint,
ApiToken: configs.Configuration.InternalApisAuthToken,
HostName: hostname,
}
sm := manager.NewSyncManager(
cliFlags,
blobsDir,
configs.Configuration.LbrycrdString,
apiConfig,
)
err = sm.Start()
if err != nil {
ytUtils.SendErrorToSlack(errors.FullTrace(err))
}
ytUtils.SendInfoToSlack("Syncing process terminated!")
}
+4 -12
View File
@@ -24,18 +24,16 @@ type SyncManager struct {
CliFlags shared.SyncFlags
ApiConfig *sdk.APIConfig
LbrycrdDsn string
AwsConfigs *shared.AwsConfigs
blobsDir string
channelsToSync []Sync
}
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, apiConfig *sdk.APIConfig) *SyncManager {
return &SyncManager{
CliFlags: cliFlags,
blobsDir: blobsDir,
LbrycrdDsn: lbrycrdDsn,
AwsConfigs: awsConfigs,
ApiConfig: apiConfig,
}
}
@@ -58,12 +56,9 @@ func (s *SyncManager) Start() error {
}
}
var (
lastChannelProcessed string
secondLastChannelProcessed string
syncCount int
)
var lastChannelProcessed string
var secondLastChannelProcessed string
syncCount := 0
for {
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
err := s.checkUsedSpace()
@@ -111,12 +106,10 @@ func (s *SyncManager) Start() error {
log.Infof("Drained the \"%s\" queue", q)
}
}
if len(s.channelsToSync) == 0 {
log.Infoln("No channels to sync. Pausing 5 minutes!")
time.Sleep(5 * time.Minute)
}
for _, sync := range s.channelsToSync {
if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
@@ -179,7 +172,6 @@ func (s *SyncManager) Start() error {
break
}
}
return nil
}
+61 -40
View File
@@ -3,6 +3,12 @@ package manager
import (
"os"
"path/filepath"
"strings"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -10,24 +16,21 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
logUtils "github.com/lbryio/ytsync/v5/util"
)
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
func (s *Sync) getS3Downloader(config *aws.Config) (*s3manager.Downloader, error) {
s3Session, err := session.NewSession(config)
if err != nil {
return nil, errors.Prefix("error starting session: ", err)
return nil, errors.Prefix("error starting session", err)
}
downloader := s3manager.NewDownloader(s3Session)
return downloader, nil
}
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
func (s *Sync) getS3Uploader(config *aws.Config) (*s3manager.Uploader, error) {
s3Session, err := session.NewSession(config)
if err != nil {
return nil, errors.Prefix("error starting session: ", err)
return nil, errors.Prefix("error starting session", err)
}
uploader := s3manager.NewUploader(s3Session)
return uploader, nil
@@ -38,18 +41,18 @@ func (s *Sync) downloadWallet() error {
if err != nil {
return errors.Err(err)
}
downloader, err := s.getS3Downloader()
downloader, err := s.getS3Downloader(configs.Configuration.WalletS3Config.GetS3AWSConfig())
if err != nil {
return err
}
out, err := os.Create(defaultTempWalletDir)
if err != nil {
return errors.Prefix("error creating temp wallet: ", err)
return errors.Prefix("error creating temp wallet", err)
}
defer out.Close()
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Key: key,
})
if err != nil {
@@ -74,21 +77,21 @@ func (s *Sync) downloadWallet() error {
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
if err != nil {
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
return errors.Prefix("error replacing temp wallet for default wallet", err)
}
return nil
}
func (s *Sync) downloadBlockchainDB() error {
if logUtils.IsRegTest() {
if util.IsRegTest() {
return nil // tests fail if we re-use the same blockchain DB
}
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
defaultBDBPath, defaultTempBDBPath, key, err := s.getBlockchainDBPaths()
if err != nil {
return errors.Err(err)
}
files, err := filepath.Glob(defaultBDBDir + "*")
files, err := filepath.Glob(defaultBDBPath + "*")
if err != nil {
return errors.Err(err)
}
@@ -101,18 +104,18 @@ func (s *Sync) downloadBlockchainDB() error {
if s.DbChannelData.WipeDB {
return nil
}
downloader, err := s.getS3Downloader()
downloader, err := s.getS3Downloader(configs.Configuration.BlockchaindbS3Config.GetS3AWSConfig())
if err != nil {
return errors.Err(err)
}
out, err := os.Create(defaultTempBDBDir)
out, err := os.Create(defaultTempBDBPath)
if err != nil {
return errors.Prefix("error creating temp wallet: ", err)
return errors.Prefix("error creating temp blockchain DB file", err)
}
defer out.Close()
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Bucket: aws.String(configs.Configuration.BlockchaindbS3Config.Bucket),
Key: key,
})
if err != nil {
@@ -135,11 +138,16 @@ func (s *Sync) downloadBlockchainDB() error {
return errors.Err("zero bytes written")
}
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
blockchainDbDir := strings.Replace(defaultBDBPath, "blockchain.db", "", -1)
err = util.Untar(defaultTempBDBPath, blockchainDbDir)
if err != nil {
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
return errors.Prefix("error extracting blockchain.db files", err)
}
log.Printf("blockchain.db downloaded to %s", defaultBDBDir)
err = os.Remove(defaultTempBDBPath)
if err != nil {
return errors.Err(err)
}
log.Printf("blockchain.db data downloaded and extracted to %s", blockchainDbDir)
return nil
}
@@ -147,7 +155,7 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string,
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
if util.IsRegTest() {
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
@@ -168,27 +176,27 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string,
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir == "" {
if logUtils.IsRegTest() {
if util.IsRegTest() {
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
} else {
lbryumDir = os.Getenv("HOME") + "/.lbryum"
}
}
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.tar"
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId + ".tar")
if util.IsRegTest() {
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.tar"
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId + ".tar")
}
return
}
func (s *Sync) uploadWallet() error {
defaultWalletDir := logUtils.GetDefaultWalletPath()
defaultWalletDir := util.GetDefaultWalletPath()
key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
if util.IsRegTest() {
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
}
@@ -196,7 +204,7 @@ func (s *Sync) uploadWallet() error {
return errors.Err("default_wallet does not exist")
}
uploader, err := s.getS3Uploader()
uploader, err := s.getS3Uploader(configs.Configuration.WalletS3Config.GetS3AWSConfig())
if err != nil {
return err
}
@@ -208,7 +216,7 @@ func (s *Sync) uploadWallet() error {
defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Key: key,
Body: file,
})
@@ -229,26 +237,39 @@ func (s *Sync) uploadBlockchainDB() error {
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
return errors.Err("blockchain.db does not exist")
}
uploader, err := s.getS3Uploader()
files, err := filepath.Glob(defaultBDBDir + "*")
if err != nil {
return errors.Err(err)
}
tarPath := strings.Replace(defaultBDBDir, "blockchain.db", "", -1) + s.DbChannelData.ChannelId + ".tar"
err = util.CreateTarball(tarPath, files)
if err != nil {
return err
}
file, err := os.Open(defaultBDBDir)
uploader, err := s.getS3Uploader(configs.Configuration.BlockchaindbS3Config.GetS3AWSConfig())
if err != nil {
return err
}
file, err := os.Open(tarPath)
if err != nil {
return err
}
defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Bucket: aws.String(configs.Configuration.BlockchaindbS3Config.Bucket),
Key: key,
Body: file,
})
if err != nil {
return err
}
log.Println("blockchain.db uploaded to S3")
log.Println("blockchain.db files uploaded to S3")
err = os.Remove(tarPath)
if err != nil {
return errors.Err(err)
}
return os.Remove(defaultBDBDir)
}
+3 -4
View File
@@ -311,12 +311,12 @@ func (s *Sync) waitForNewBlock() error {
func (s *Sync) GenerateRegtestBlock() error {
lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
return errors.Prefix("error getting lbrycrd client", err)
}
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block: ", err)
return errors.Prefix("error generating new block", err)
}
for _, tx := range txs {
@@ -407,7 +407,7 @@ func (s *Sync) ensureChannelOwnership() error {
}
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId)
if err != nil {
return err
}
@@ -416,7 +416,6 @@ func (s *Sync) ensureChannelOwnership() error {
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
"banner-"+s.DbChannelData.ChannelId,
*s.Manager.AwsConfigs.GetS3AWSConfig(),
)
if err != nil {
return err
+11 -5
View File
@@ -286,6 +286,8 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
"interrupted during daemon startup",
"interrupted by user",
"use --skip-space-check to ignore",
"failure uploading blockchain DB",
"default_wallet already exists",
}
dbWipeConditions := []string{
"Missing inputs",
@@ -348,19 +350,24 @@ func (s *Sync) stopAndUploadWallet(e *error) {
logShutdownError(processDeathError)
} else {
err := s.uploadWallet()
if err != nil {
time.Sleep(10 * time.Second)
logUtils.SendErrorToSlack("there was a problem uploading the wallet to S3, waiting 10 seconds and retrying: %s", err.Error())
err = s.uploadWallet()
}
if err != nil {
if *e == nil {
e = &err
*e = err
} else {
*e = errors.Prefix("failure uploading wallet", *e)
*e = errors.Prefix(fmt.Sprintf("failure uploading wallet: %s + original error", errors.FullTrace(err)), *e)
}
}
err = s.uploadBlockchainDB()
if err != nil {
if *e == nil {
e = &err
*e = err
} else {
*e = errors.Prefix("failure uploading wallet", *e)
*e = errors.Prefix(fmt.Sprintf("failure uploading blockchain DB: %s + original error", errors.FullTrace(err)), *e)
}
}
}
@@ -863,7 +870,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
VideoDir: s.videoDirectory,
S3Config: *s.Manager.AwsConfigs.GetS3AWSConfig(),
Stopper: s.grp,
IPPool: ipPool,
}, s.DbChannelData.LastUploadedVideo)
+3 -4
View File
@@ -25,10 +25,9 @@ const (
)
type APIConfig struct {
YoutubeAPIKey string
ApiURL string
ApiToken string
HostName string
ApiURL string
ApiToken string
HostName string
}
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
-16
View File
@@ -4,8 +4,6 @@ import (
"encoding/json"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
@@ -213,17 +211,3 @@ const (
TransferStateComplete
TransferStateManual
)
type AwsConfigs struct {
AwsS3ID string
AwsS3Secret string
AwsS3Region string
AwsS3Bucket string
}
func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
return &aws.Config{
Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
Region: &a.AwsS3Region,
}
}
+5 -8
View File
@@ -37,7 +37,6 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
)
@@ -55,7 +54,6 @@ type YoutubeVideo struct {
youtubeInfo *ytdl.YtdlVideo
youtubeChannelID string
tags []string
awsConfig aws.Config
thumbnailURL string
lbryChannelID string
mocked bool
@@ -101,7 +99,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
// youtube-dl returns times in local timezone sometimes. this could break in the future
// maybe we can file a PR to choose the timezone we want from youtube-dl
return &YoutubeVideo{
@@ -112,19 +110,18 @@ func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPositi
publishedAt: videoData.UploadDateForReal,
dir: directory,
youtubeInfo: videoData,
awsConfig: awsConfig,
mocked: false,
youtubeChannelID: videoData.ChannelID,
stopGroup: stopGroup,
pool: pool,
}, nil
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
dir: directory,
awsConfig: awsConfig,
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
@@ -688,7 +685,7 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
if thumbnail.Width == 0 {
return errors.Err("default youtube thumbnail found")
}
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID())
return err
}
@@ -880,7 +877,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err("could not find thumbnail for mocked video")
}
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID())
} else {
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
}
+5 -6
View File
@@ -6,6 +6,7 @@ import (
"os"
"strings"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -83,11 +84,11 @@ func (u *thumbnailUploader) deleteTmpFile() {
log.Infof("failed to delete local thumbnail file: %s", err.Error())
}
}
func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, error) {
func MirrorThumbnail(url string, name string) (string, error) {
tu := thumbnailUploader{
originalUrl: url,
name: name,
s3Config: s3Config,
s3Config: *configs.Configuration.AWSThumbnailsS3Config.GetS3AWSConfig(),
}
err := tu.downloadThumbnail()
if err != nil {
@@ -100,14 +101,12 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
return "", err
}
ownS3Config := s3Config.Copy(&aws.Config{Endpoint: aws.String("s3.lbry.tech")})
//this is our own S3 storage
tu2 := thumbnailUploader{
originalUrl: url,
name: name,
s3Config: *ownS3Config,
s3Config: *configs.Configuration.ThumbnailsS3Config.GetS3AWSConfig(),
}
//own S3
err = tu2.uploadThumbnail()
if err != nil {
return "", err
+109
View File
@@ -0,0 +1,109 @@
package util
import (
"archive/tar"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
func CreateTarball(tarballFilePath string, filePaths []string) error {
file, err := os.Create(tarballFilePath)
if err != nil {
return errors.Err("Could not create tarball file '%s', got error '%s'", tarballFilePath, err.Error())
}
defer file.Close()
tarWriter := tar.NewWriter(file)
defer tarWriter.Close()
for _, filePath := range filePaths {
err := addFileToTarWriter(filePath, tarWriter)
if err != nil {
return errors.Err("Could not add file '%s', to tarball, got error '%s'", filePath, err.Error())
}
}
return nil
}
func addFileToTarWriter(filePath string, tarWriter *tar.Writer) error {
file, err := os.Open(filePath)
if err != nil {
return errors.Err("Could not open file '%s', got error '%s'", filePath, err.Error())
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return errors.Err("Could not get stat for file '%s', got error '%s'", filePath, err.Error())
}
header := &tar.Header{
Name: stat.Name(),
Size: stat.Size(),
Mode: int64(stat.Mode()),
ModTime: stat.ModTime(),
}
err = tarWriter.WriteHeader(header)
if err != nil {
return errors.Err("Could not write header for file '%s', got error '%s'", filePath, err.Error())
}
_, err = io.Copy(tarWriter, file)
if err != nil {
return errors.Err("Could not copy the file '%s' data to the tarball, got error '%s'", filePath, err.Error())
}
return nil
}
func Untar(tarball, target string) error {
reader, err := os.Open(tarball)
if err != nil {
return errors.Err(err)
}
defer reader.Close()
tarReader := tar.NewReader(reader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
return errors.Err(err)
}
path := filepath.Join(target, header.Name)
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return errors.Err(err)
}
continue
}
err = extractFile(path, info, tarReader)
if err != nil {
return err
}
}
return nil
}
func extractFile(path string, info fs.FileInfo, tarReader *tar.Reader) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
return errors.Err(err)
}
defer file.Close()
_, err = io.Copy(file, tarReader)
if err != nil {
return errors.Err(err)
}
return nil
}
+5 -1
View File
@@ -14,7 +14,9 @@ func SendErrorToSlack(format string, a ...interface{}) {
message = fmt.Sprintf(format, a...)
}
log.Errorln(message)
err := util.SendToSlack(":sos: " + message)
log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
err := util.SendToSlack(":sos: ```" + message + "```")
log.SetLevel(log.DebugLevel)
if err != nil {
log.Errorln(err)
}
@@ -27,7 +29,9 @@ func SendInfoToSlack(format string, a ...interface{}) {
message = fmt.Sprintf(format, a...)
}
log.Infoln(message)
log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
err := util.SendToSlack(":information_source: " + message)
log.SetLevel(log.DebugLevel)
if err != nil {
log.Errorln(err)
}
+3 -2
View File
@@ -11,6 +11,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/timing"
"github.com/docker/docker/api/types"
@@ -185,9 +186,9 @@ func CleanForStartup() error {
return errors.Err(err)
}
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
lbrycrd, err := GetLbrycrdClient(configs.Configuration.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
return errors.Prefix("error getting lbrycrd client", err)
}
height, err := lbrycrd.GetBlockCount()
if err != nil {
+2 -4
View File
@@ -29,7 +29,6 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/aws/aws-sdk-go/aws"
log "github.com/sirupsen/logrus"
)
@@ -50,7 +49,6 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
type VideoParams struct {
VideoDir string
S3Config aws.Config
Stopper *stop.Group
IPPool *ip_manager.IPPool
}
@@ -103,7 +101,7 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
for _, item := range vids {
positionInList := playlistMap[item.ID]
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.Stopper, videoParams.IPPool)
if err != nil {
return nil, errors.Err(err)
}
@@ -115,7 +113,7 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
continue
}
if _, ok := playlistMap[k]; !ok {
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.Stopper, videoParams.IPPool))
}
}