Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce35994a41 | ||
|
|
ef950997e5 | ||
|
|
aecdc1c29f | ||
|
|
0d8da62736 |
No files matched your search
@@ -1,13 +1,103 @@
|
||||
# YT Sync Process
|
||||
# YTSync Tool
|
||||
[](https://travis-ci.com/lbryio/ytsync)
|
||||
|
||||
- make sure you don't have a `.lbryum/wallets/default_wallet`
|
||||
- delete existing wallet if there's nothing you need there, or better yet, move it somewhere else in case you need it later
|
||||
- make sure daemon is stopped and can be controlled with `systemctl`
|
||||
- run `lbry ytsync YOUTUBE_KEY LBRY_CHANNEL_NAME YOUTUBE_CHANNEL_ID`
|
||||
- after sync is complete, daemon will be stopped and wallet will be moved to `~/wallets/`
|
||||
- now mark content as synced in doc
|
||||
This tool serves lbry to parse youtube channels that want their content mirrored on LBRY.
|
||||
|
||||
Running the sync command for a channel that was already started will resume the sync. This can also be used to update a channel with new
|
||||
content that was put on Youtube since the last sync.
|
||||
The tool downloads the entire set of public videos from a given channel, publishes them to LBRY and populates our private database in order to keep track of what's publishes.
|
||||
With the support of said database, the tool is also able to keep all the channels updated.
|
||||
|
||||
---
|
||||
|
||||
# Requirements
|
||||
- lbrynet SDK https://github.com/lbryio/lbry/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
|
||||
- a lbrycrd node running (localhost or on a remote machine) with credits in it
|
||||
|
||||
# Setup
|
||||
- make sure daemon is stopped and can be controlled through `systemctl` (find example below)
|
||||
- extract the ytsync binary anywhere
|
||||
- add the environment variables necessary to the tool
|
||||
- export SLACK_TOKEN="a-token-to-spam-your-slack"
|
||||
- export SLACK_CHANNEL="youtube-status"
|
||||
- export YOUTUBE_API_KEY="youtube-api-key"
|
||||
- export LBRY_WEB_API="https://lbry-api-url-here"
|
||||
- export LBRY_API_TOKEN="internal-apis-token-for-ytsync-user"
|
||||
- export LBRYCRD_STRING="tcp://user:password@host:5429"
|
||||
- export AWS_S3_ID="THE-ID-LIES-HERE"
|
||||
- export AWS_S3_SECRET="THE-SECRET-LIES-HERE"
|
||||
- export AWS_S3_REGION="us-east-1"
|
||||
- export AWS_S3_BUCKET="ytsync-wallets"
|
||||
|
||||
## systemd script example
|
||||
`/etc/systemd/system/lbrynet.service`
|
||||
```
|
||||
[Unit]
|
||||
Description="LBRYnet daemon"
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Environment="HOME=/home/lbry"
|
||||
ExecStart=/opt/lbry/lbrynet start
|
||||
User=lbry
|
||||
Group=lbry
|
||||
Restart=on-failure
|
||||
KillMode=process
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
# Instructions
|
||||
|
||||
```
|
||||
Publish youtube channels into LBRY network automatically.
|
||||
|
||||
Usage:
|
||||
ytsync [flags]
|
||||
|
||||
Flags:
|
||||
--after int Specify from when to pull jobs [Unix time](Default: 0)
|
||||
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1582834707)
|
||||
--channelID string If specified, only this channel will be synced.
|
||||
--concurrent-jobs int how many jobs to process concurrently (default 1)
|
||||
-h, --help help for ytsync
|
||||
--limit int limit the amount of channels to sync
|
||||
--max-length float Maximum video length to process (in hours) (default 2)
|
||||
--max-size int Maximum video size to process (in MB) (default 2048)
|
||||
--max-tries int Number of times to try a publish that fails (default 3)
|
||||
--run-once Whether the process should be stopped after one cycle or not
|
||||
--skip-space-check Do not perform free space check on startup
|
||||
--status string Specify which queue to pull from. Overrides --update
|
||||
--stop-on-error If a publish fails, stop all publishing and exit
|
||||
--takeover-existing-channel If channel exists and we don't own it, take over the channel
|
||||
--update Update previously synced channels instead of syncing new ones
|
||||
--videos-limit int how many videos to process per channel (default 1000)
|
||||
```
|
||||
|
||||
## Running from Source
|
||||
|
||||
Clone the repository and run `make`
|
||||
|
||||
## License
|
||||
|
||||
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, see [CONTRIBUTING](https://lbry.tech/contribute).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.io).
|
||||
|
||||
## Additional Info and Links
|
||||
|
||||
- [https://lbry.io](https://lbry.io) - The live LBRY website
|
||||
- [Discord Chat](https://chat.lbry.io) - A chat room for the LBRYians
|
||||
- [Email us](mailto:hello@lbry.io) - LBRY Support email
|
||||
- [Twitter](https://twitter.com/@lbryio) - LBRY Twitter page
|
||||
- [Facebook](https://www.facebook.com/lbryio/) - LBRY Facebook page
|
||||
- [Reddit](https://reddit.com/r/lbry) - LBRY Reddit page
|
||||
- [Telegram](https://t.me/lbryofficial) - Telegram group
|
||||
@@ -8,7 +8,7 @@ require (
|
||||
github.com/gorilla/websocket v1.4.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/lbry.go v0.0.0-20190222180913-9b603ca498c6
|
||||
github.com/lbryio/lbry.go v0.0.0-20190321133538-94802df513b0
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
|
||||
|
||||
@@ -45,6 +45,8 @@ github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200j
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
@@ -76,11 +78,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190222180913-9b603ca498c6 h1:llfL4wNCmPiu97HFeoDlrGPwjRgIDGuJJNzeP0Nz4bQ=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190222180913-9b603ca498c6/go.mod h1:NEYqeulfWa6bW9sajq5zdd1cAJ+o8v4j1aFsqvWmySs=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190321133538-94802df513b0 h1:fg5aN9eMFV4KcTsOCXSZmIqTBHei2c2z/aWZ3ZHE1zo=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190321133538-94802df513b0/go.mod h1:8Cl5i+Rmi6gMr0VoxRjaZivMK6OKA3QUzr05MgoQb6c=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00 h1:1qRpd8lcyVigX+kYkwQL13gpOURyytgvxZtuIQfPPX8=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190304154608-a6f2fd6d5f38 h1:78rErXiBhMjs7vh4Ke3ZMqQjf+pnQeNrkXYmQ8L6aKE=
|
||||
github.com/lbryio/types v0.0.0-20190304154608-a6f2fd6d5f38/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
|
||||
|
||||
+9
-8
@@ -70,16 +70,17 @@ func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
|
||||
Reference in new issue
Block a user