Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
75628d8530 | ||
|
|
6e819b20f6 | ||
|
|
da0b6e5b79 | ||
|
|
28791f317b |
No files matched your search
@@ -4,3 +4,7 @@ e2e/supporty/supporty
|
||||
.env
|
||||
blobsfiles
|
||||
ytsync_docker
|
||||
|
||||
e2e/config.json
|
||||
|
||||
e2e/cookies.txt
|
||||
@@ -8,23 +8,17 @@ With the support of said database, the tool is also able to keep all the channel
|
||||
|
||||
|
||||
# Requirements
|
||||
- lbrynet SDK https://github.com/lbryio/lbry/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
|
||||
- lbrynet SDK https://github.com/lbryio/lbry-sdk/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
|
||||
- a lbrycrd node running (localhost or on a remote machine) with credits in it
|
||||
- internal-apis (you cannot run this one yourself)
|
||||
- python3-pip
|
||||
- yt-dlp (`pip3 install -U yt-dlp`)
|
||||
- ffmpeg (latest)
|
||||
|
||||
# Setup
|
||||
- make sure daemon is stopped and can be controlled through `systemctl` (find example below)
|
||||
- extract the ytsync binary anywhere
|
||||
- add the environment variables necessary to the tool
|
||||
- export SLACK_TOKEN="a-token-to-spam-your-slack"
|
||||
- export SLACK_CHANNEL="youtube-status"
|
||||
- export YOUTUBE_API_KEY="youtube-api-key"
|
||||
- export LBRY_WEB_API="https://lbry-api-url-here"
|
||||
- export LBRY_API_TOKEN="internal-apis-token-for-ytsync-user"
|
||||
- export LBRYCRD_STRING="tcp://user:password@host:5429"
|
||||
- export AWS_S3_ID="THE-ID-LIES-HERE"
|
||||
- export AWS_S3_SECRET="THE-SECRET-LIES-HERE"
|
||||
- export AWS_S3_REGION="us-east-1"
|
||||
- export AWS_S3_BUCKET="ytsync-wallets"
|
||||
- create and fill `config.json` using [this example](config.json.example)
|
||||
|
||||
## systemd script example
|
||||
`/etc/systemd/system/lbrynet.service`
|
||||
@@ -55,23 +49,26 @@ Usage:
|
||||
|
||||
Flags:
|
||||
--after int Specify from when to pull jobs [Unix time](Default: 0)
|
||||
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
|
||||
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1669311891)
|
||||
--channelID string If specified, only this channel will be synced.
|
||||
--concurrent-jobs int how many jobs to process concurrently (default 1)
|
||||
-h, --help help for ytsync
|
||||
--limit int limit the amount of channels to sync
|
||||
--max-length float Maximum video length to process (in hours) (default 2)
|
||||
--max-length int Maximum video length to process (in hours) (default 2)
|
||||
--max-size int Maximum video size to process (in MB) (default 2048)
|
||||
--max-tries int Number of times to try a publish that fails (default 3)
|
||||
--no-transfers Skips the transferring process of videos, channels and supports
|
||||
--quick Look up only the last 50 videos from youtube
|
||||
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
|
||||
--run-once Whether the process should be stopped after one cycle or not
|
||||
--skip-space-check Do not perform free space check on startup
|
||||
--status string Specify which queue to pull from. Overrides --update
|
||||
--stop-on-error If a publish fails, stop all publishing and exit
|
||||
--status2 string Specify which secondary queue to pull from.
|
||||
--takeover-existing-channel If channel exists and we don't own it, take over the channel
|
||||
--update Update previously synced channels instead of syncing new ones
|
||||
--upgrade-metadata Upgrade videos if they're on the old metadata version
|
||||
--videos-limit int how many videos to process per channel (default 1000)
|
||||
--videos-limit int how many videos to process per channel (leave 0 for automatic detection)
|
||||
|
||||
```
|
||||
|
||||
## Running from Source
|
||||
@@ -92,13 +89,13 @@ We take security seriously. Please contact [security@lbry.io](mailto:security@lb
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.io).
|
||||
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.com).
|
||||
|
||||
## Additional Info and Links
|
||||
|
||||
- [https://lbry.io](https://lbry.io) - The live LBRY website
|
||||
- [Discord Chat](https://chat.lbry.io) - A chat room for the LBRYians
|
||||
- [Email us](mailto:hello@lbry.io) - LBRY Support email
|
||||
- [https://lbry.com](https://lbry.com) - The live LBRY website
|
||||
- [Discord Chat](https://chat.lbry.com) - A chat room for the LBRYians
|
||||
- [Email us](mailto:hello@lbry.com) - LBRY Support email
|
||||
- [Twitter](https://twitter.com/@lbryio) - LBRY Twitter page
|
||||
- [Facebook](https://www.facebook.com/lbryio/) - LBRY Facebook page
|
||||
- [Reddit](https://reddit.com/r/lbry) - LBRY Reddit page
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"slack_token": "",
|
||||
"slack_channel": "ytsync-dev",
|
||||
"internal_apis_endpoint": "http://localhost:15400",
|
||||
"internal_apis_auth_token": "ytsyntoken",
|
||||
"lbrycrd_string": "tcp://lbry:lbry@localhost:15200",
|
||||
"wallet_s3_config": {
|
||||
"id": "",
|
||||
"secret": "",
|
||||
"region": "us-east-1",
|
||||
"bucket": "ytsync-wallets",
|
||||
"endpoint": ""
|
||||
},
|
||||
"blockchaindb_s3_config": {
|
||||
"id": "",
|
||||
"secret": "",
|
||||
"region": "us-east-1",
|
||||
"bucket": "blockchaindbs",
|
||||
"endpoint": ""
|
||||
},
|
||||
"thumbnails_s3_config": {
|
||||
"id": "",
|
||||
"secret": "",
|
||||
"region": "us-east-1",
|
||||
"bucket": "thumbnails.lbry.com",
|
||||
"endpoint": ""
|
||||
},
|
||||
"aws_thumbnails_s3_config": {
|
||||
"id": "",
|
||||
"secret": "",
|
||||
"region": "us-east-1",
|
||||
"bucket": "thumbnails.lbry.com",
|
||||
"endpoint": ""
|
||||
}
|
||||
}
|
||||
+11
-4
@@ -84,9 +84,9 @@ func (s *Sync) downloadWallet() error {
|
||||
}
|
||||
|
||||
func (s *Sync) downloadBlockchainDB() error {
|
||||
//if util.IsRegTest() {
|
||||
// return nil // tests fail if we re-use the same blockchain DB
|
||||
//}
|
||||
if util.IsRegTest() {
|
||||
return nil // tests fail if we re-use the same blockchain DB
|
||||
}
|
||||
defaultBDBPath, defaultTempBDBPath, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -143,7 +143,10 @@ func (s *Sync) downloadBlockchainDB() error {
|
||||
if err != nil {
|
||||
return errors.Prefix("error extracting blockchain.db files", err)
|
||||
}
|
||||
|
||||
err = os.Remove(defaultTempBDBPath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Printf("blockchain.db data downloaded and extracted to %s", blockchainDbDir)
|
||||
return nil
|
||||
}
|
||||
@@ -264,5 +267,9 @@ func (s *Sync) uploadBlockchainDB() error {
|
||||
return err
|
||||
}
|
||||
log.Println("blockchain.db files uploaded to S3")
|
||||
err = os.Remove(tarPath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return os.Remove(defaultBDBDir)
|
||||
}
|
||||
+1
-1
@@ -14,7 +14,7 @@ func SendErrorToSlack(format string, a ...interface{}) {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Errorln(message)
|
||||
err := util.SendToSlack(":sos: " + message)
|
||||
err := util.SendToSlack(":sos: ```" + message + "```")
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user