Compare commits

..
28 Commits
Author SHA1 Message Date
Niko Storni 07915ce697 fix travis try 2 2020-06-10 03:44:34 +02:00
Niko Storni 4b2221ce5e fix travis (try 1) 2020-06-10 03:38:42 +02:00
Niko Storni 1d5b69bfe6 add automatic sleep until quota reset
re-enable ipv6
2020-06-10 03:32:45 +02:00
Niko Storni 4d915587e0 temporarily disable ipv6 2020-05-30 02:18:42 +02:00
Niko Storni 83f2c28c20 fix nil ptr 2020-05-22 12:47:18 -04:00
Niko Storni 47e467057d change timeouts to avoid long wait times 2020-05-20 04:28:29 +02:00
Niko Storni 1ae32d638b fix prom allowed chars 2020-05-19 23:24:16 +02:00
Niko Storni 7907ee3579 add prometheus metrics 2020-05-19 23:13:01 +02:00
Niko Storni 414ed1c130 or to and 2020-05-14 17:28:15 +02:00
Niko Storni 3bb17b09d3 fix never ending condition 2020-05-14 06:47:19 +02:00
Niko Storni 046f46267e update lbry.go dependency 2020-05-14 05:49:49 +02:00
Niko Storni adc3ffe194 update lbrynet support 2020-05-14 04:04:52 +02:00
Niko Storni 76301d40ba only wait a maximum of 2 blocks before transfering
fix debug string
2020-05-08 16:33:29 +02:00
Niko Storni 963c450730 allow all IPv6 to be used 2020-05-05 20:13:40 +02:00
Niko Storni 6e7bb994d0 don't keep reusing the same IP for updates 2020-05-05 05:58:34 +02:00
Niko Storni 95890cbabf prevent live streams from syncing
reduce timeout to 20 minutes
2020-04-29 16:15:22 +02:00
Niko Storni 69c2a91ec7 fix bug in video length 2020-04-23 16:37:35 +02:00
Niko Storni fbe09a692c fix 403 error 2020-04-21 20:56:14 +02:00
Niko Storni 0ccc6e7979 add temporary logging 2020-04-21 16:26:34 +02:00
Niko Storni 3eb74cc8b5 log video length failures 2020-04-19 02:38:31 +02:00
Niko Storni 9bfb1656fe fix length checks on single syncs 2020-04-10 17:46:07 +02:00
Niko Storni 5d40d24804 fix max length issue for bigger youtubers
fix issue with no-playlist-found channels
reduce the max db size limit
2020-04-08 23:14:10 +02:00
Niko Storni 79d3aadda8 add utxo_release as hotfix for stuck balances 2020-04-01 04:49:24 +02:00
Niko Storni eba59a0806 change utxo_list signature 2020-04-01 04:44:05 +02:00
Niko Storni 3fd4909cd1 replace support abandon with txo_spend
upgrade lbry.go library
upgrade lbrynet support
possibly introduce new bugs ;)
2020-03-31 04:30:02 +02:00
Niko Storni b59372c1d8 don't prevent channels with 0 youtube videos from transferring 2020-03-31 02:45:53 +02:00
Niko Storni c9e713e9df do not attempt upgrading channels no longer in our control 2020-03-27 18:16:35 +01:00
Niko Storni bc72d2f1c4 do not fail if database is missing 2020-03-25 18:28:42 +01:00
18 changed files with 345 additions and 51 deletions

No files matched your search

+4 -6
View File
@@ -2,8 +2,7 @@ os: linux
dist: bionic
language: go
go:
- 1.12.7
- master
- 1.14.4
install: true
@@ -21,15 +20,14 @@ addons:
packages:
- ffmpeg
- tree
- python-pip
- python3-pip
before_script:
- sudo pip install -U youtube-dl
- sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
- sudo pip3 install -U youtube-dl
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
+2 -2
View File
@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0,1,1)"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
+2 -2
View File
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.60.0
image: lbry/wallet-server:v0.73.1
restart: always
environment:
- DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.60.0
image: lbry/lbrynet:v0.73.1
restart: always
ports:
- "15100:5279"
+1 -1
View File
@@ -8,7 +8,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.51.2
image: lbry/lbrynet:v0.73.1
restart: "no"
networks:
lbry-network:
+1 -1
View File
@@ -8,7 +8,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.38.6
image: lbry/wallet-server:v0.73.1
restart: always
networks:
lbry-network:
+2 -1
View File
@@ -20,12 +20,13 @@ require (
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88
github.com/lbryio/lbry.go/v2 v2.6.0
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
github.com/prometheus/client_golang v0.9.2
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
+8 -2
View File
@@ -29,6 +29,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
@@ -188,8 +189,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88 h1:SOn28lm4Bd8RY63hFVyt0d6SAD9OtFMEJVaKrO+pc/E=
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
github.com/lbryio/lbry.go/v2 v2.6.0 h1:qC2NsSZH2AeNDOP0Dw62cKNqHPy1eEDladmgnhOrIQc=
github.com/lbryio/lbry.go/v2 v2.6.0/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
@@ -208,6 +209,7 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCW
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
@@ -248,9 +250,13 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+1 -4
View File
@@ -43,15 +43,13 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
return nil, errors.Err(err)
}
var pool []throttledIP
ipv6Added := false
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
ipv6Added = true
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
@@ -130,7 +128,6 @@ func (i *IPPool) ReleaseAll() {
}
localIP := &i.ips[j]
localIP.InUse = false
localIP.LastUse = time.Now()
}
}
+6 -1
View File
@@ -3,6 +3,7 @@ package main
import (
"fmt"
"math/rand"
"net/http"
"os"
"time"
@@ -11,6 +12,7 @@ import (
"github.com/lbryio/ytsync/manager"
"github.com/lbryio/ytsync/sdk"
ytUtils "github.com/lbryio/ytsync/util"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -38,7 +40,10 @@ var (
func main() {
rand.Seed(time.Now().UnixNano())
log.SetLevel(log.DebugLevel)
http.Handle("/metrics", promhttp.Handler())
go func() {
log.Error(http.ListenAndServe(":2112", nil))
}()
cmd := &cobra.Command{
Use: "ytsync",
Short: "Publish youtube channels into LBRY network automatically.",
+8 -2
View File
@@ -135,6 +135,7 @@ func (s *SyncManager) Start() error {
ConcurrentVideos: s.concurrentVideos,
Refill: s.refill,
Manager: s,
MaxVideoLength: s.maxVideoLength,
LbrycrdString: s.lbrycrdString,
AwsS3ID: s.awsS3ID,
AwsS3Secret: s.awsS3Secret,
@@ -168,8 +169,9 @@ func (s *SyncManager) Start() error {
}
for i, c := range channels {
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
maxVideoLength := s.maxVideoLength
if c.TotalSubscribers < 1000 {
s.maxVideoLength = 1.0
maxVideoLength = 1.0
}
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
@@ -178,6 +180,7 @@ func (s *SyncManager) Start() error {
lbryChannelID: c.ChannelClaimID,
MaxTries: s.maxTries,
ConcurrentVideos: s.concurrentVideos,
MaxVideoLength: maxVideoLength,
Refill: s.refill,
Manager: s,
LbrycrdString: s.lbrycrdString,
@@ -212,6 +215,9 @@ func (s *SyncManager) Start() error {
}
if err != nil {
if strings.Contains(err.Error(), "quotaExceeded") {
logUtils.SleepUntilQuotaReset()
}
fatalErrors := []string{
"default_wallet already exists",
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
@@ -221,8 +227,8 @@ func (s *SyncManager) Start() error {
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
"Daily Limit Exceeded",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
}
+40 -10
View File
@@ -10,6 +10,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/timing"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/tags_manager"
@@ -49,6 +50,10 @@ func (s *Sync) enableAddressReuse() error {
return nil
}
func (s *Sync) walletSetup() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("walletSetup").Add(time.Since(start))
}(start)
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
s.walletMux.Lock()
defer s.walletMux.Unlock()
@@ -151,6 +156,10 @@ func (s *Sync) walletSetup() error {
}
func (s *Sync) getDefaultAccount() (string, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
}(start)
if s.defaultAccountID == "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
@@ -177,6 +186,10 @@ func (s *Sync) getDefaultAccount() (string, error) {
}
func (s *Sync) ensureEnoughUTXOs() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
}(start)
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
@@ -196,7 +209,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
for _, utxo := range utxolist.Items {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
confirmedCount++
}
@@ -253,6 +266,10 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
func (s *Sync) waitForNewBlock() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
}(start)
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
status, err := s.daemon.Status()
if err != nil {
@@ -302,6 +319,10 @@ func (s *Sync) GenerateRegtestBlock() error {
}
func (s *Sync) ensureChannelOwnership() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
}(start)
if s.LbryChannelName == "" {
return errors.Err("no channel name set")
}
@@ -422,15 +443,20 @@ func (s *Sync) ensureChannelOwnership() error {
ThumbnailURL: &thumbnailURL,
}
if channelUsesOldMetadata {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
},
})
if s.transferState <= 1 {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
},
})
} else {
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
return nil
}
} else {
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
@@ -446,6 +472,10 @@ func (s *Sync) ensureChannelOwnership() error {
}
func (s *Sync) addCredits(amountToAdd float64) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("addCredits").Add(time.Since(start))
}(start)
log.Printf("Adding %f credits", amountToAdd)
lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
+41 -8
View File
@@ -10,20 +10,25 @@ import (
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/timing"
log "github.com/sirupsen/logrus"
)
func waitConfirmations(s *Sync) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("waitConfirmations").Add(time.Since(start))
}(start)
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
allConfirmed := false
waitCount := 0
waiting:
for !allConfirmed {
for !allConfirmed && waitCount < 2 {
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
if err != nil {
return err
@@ -37,6 +42,7 @@ waiting:
if err != nil {
return err
}
waitCount++
continue waiting
}
}
@@ -52,6 +58,10 @@ type abandonResponse struct {
}
func abandonSupports(s *Sync) (float64, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("abandonSupports").Add(time.Since(start))
}(start)
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
defaultAccount, err := s.getDefaultAccount()
@@ -85,22 +95,23 @@ func abandonSupports(s *Sync) (float64, error) {
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
defer s.daemon.SetRPCTimeout(40 * time.Minute)
s.daemon.SetRPCTimeout(60 * time.Second)
defer s.daemon.SetRPCTimeout(5 * time.Minute)
for i := 0; i < s.ConcurrentVideos; i++ {
consumerWG.Add(1)
go func() {
defer consumerWG.Done()
outer:
for {
claimID, more := <-claimIDChan
if !more {
return
} else {
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
@@ -119,7 +130,7 @@ func abandonSupports(s *Sync) (float64, error) {
continue
}
}
if len(summary.Outputs) < 1 {
if summary == nil || len(*summary) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
@@ -127,7 +138,19 @@ func abandonSupports(s *Sync) (float64, error) {
}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
var outputAmount float64
for _, tx := range *summary {
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue outer
}
outputAmount += amount
}
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
@@ -170,6 +193,10 @@ type updateInfo struct {
}
func transferVideos(s *Sync) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("transferVideos").Add(time.Since(start))
}(start)
cleanTransfer := true
streamChan := make(chan updateInfo, s.ConcurrentVideos)
@@ -258,7 +285,9 @@ func transferVideos(s *Sync) error {
}
func (s *Sync) streamUpdate(ui *updateInfo) error {
start := time.Now()
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
if updateError != nil {
ui.videoStatus.FailureReason = updateError.Error()
ui.videoStatus.Status = VideoStatusTranferFailed
@@ -275,6 +304,10 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
}
func transferChannel(s *Sync) error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("transferChannel").Add(time.Since(start))
}(start)
account, err := s.getDefaultAccount()
if err != nil {
return err
+36 -9
View File
@@ -19,6 +19,7 @@ import (
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/sources"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/ytsync/timing"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -91,6 +92,7 @@ type Sync struct {
clientPublishAddress string
publicKey string
defaultAccountID string
MaxVideoLength float64
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -241,7 +243,7 @@ func (s *Sync) setStatusSyncing() error {
func (s *Sync) setExceptions() {
if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
s.Manager.maxVideoLength = 0.0 // skips max length checks
s.MaxVideoLength = 9999.0 // skips max length checks
s.Manager.maxVideoSize = 0
}
}
@@ -257,7 +259,7 @@ func (s *Sync) FullCycle() (e error) {
}
s.setExceptions()
defer timing.ClearTimings()
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stopGroup
@@ -276,7 +278,6 @@ func (s *Sync) FullCycle() (e error) {
}
defer s.setChannelTerminationStatus(&e)
err = s.downloadWallet()
if err != nil && err.Error() != "wallet not on S3" {
return errors.Prefix("failure in downloading wallet", err)
@@ -306,7 +307,7 @@ func (s *Sync) FullCycle() (e error) {
log.Infoln("Waiting for daemon to finish starting...")
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
s.daemon.SetRPCTimeout(40 * time.Minute)
s.daemon.SetRPCTimeout(5 * time.Minute)
err = s.waitForDaemonStart()
if err != nil {
@@ -319,10 +320,10 @@ func (s *Sync) FullCycle() (e error) {
}
if s.shouldTransfer() {
return s.processTransfers()
err = s.processTransfers()
}
return nil
timing.Report()
return err
}
func (s *Sync) processTransfers() (e error) {
@@ -427,6 +428,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
func (s *Sync) waitForDaemonStart() error {
beginTime := time.Now()
defer func(start time.Time) {
timing.TimedComponent("waitForDaemonStart").Add(time.Since(start))
}(beginTime)
for {
select {
case <-s.grp.Ch():
@@ -504,6 +508,10 @@ func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
// fixDupes abandons duplicate claims
func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("fixDupes").Add(time.Since(start))
}(start)
abandonedClaims := false
videoIDs := make(map[string]jsonrpc.Claim)
for _, c := range claims {
@@ -713,6 +721,10 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
}
func (s *Sync) checkIntegrity() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("checkIntegrity").Add(time.Since(start))
}(start)
allClaims, err := s.getClaims(false)
if err != nil {
return err
@@ -781,6 +793,10 @@ func (s *Sync) doSync() error {
if err != nil {
return errors.Prefix("could not import the transferee public key", err)
}
_, err = s.daemon.UTXORelease(nil)
if err != nil {
return errors.Prefix("could not run uxo_release", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
@@ -899,6 +915,7 @@ func (s *Sync) startWorker(workerNum int) {
"interrupted by user",
"Sign in to confirm your age",
"This video is unavailable",
"video is a live stream and hasn't completed yet",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -975,7 +992,13 @@ func (s *Sync) startWorker(workerNum int) {
}
}
var mostRecentlyFailedChannel string
func (s *Sync) enqueueYoutubeVideos() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
}(start)
client := &http.Client{
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
}
@@ -1029,7 +1052,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
if youtubeIsLying {
break
}
return errors.Err("playlist items not found")
if s.YoutubeChannelID == mostRecentlyFailedChannel {
return errors.Err("playlist items not found")
}
mostRecentlyFailedChannel = s.YoutubeChannelID
break //return errors.Err("playlist items not found") //TODO: will this work?
}
videoIDs := make([]string, 50)
for i, item := range playlistResponse.Items {
@@ -1154,7 +1181,7 @@ func (s *Sync) processVideo(v video) (err error) {
ChannelID: s.lbryChannelID,
MaxVideoSize: s.Manager.maxVideoSize,
Namer: s.namer,
MaxVideoLength: s.Manager.maxVideoLength,
MaxVideoLength: s.MaxVideoLength,
Fee: s.Fee,
DefaultAccount: da,
}
+31
View File
@@ -0,0 +1,31 @@
package metrics
import (
"os"
"regexp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
)
var (
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "ytsync",
Subsystem: getHostname(),
Name: "duration",
Help: "The durations of the individual modules",
}, []string{"path"})
)
func getHostname() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "ytsync_unknown"
}
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err != nil {
log.Fatal(err)
}
return reg.ReplaceAllString(hostname, "_")
}
+1 -1
View File
@@ -146,7 +146,7 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
+20
View File
@@ -17,6 +17,8 @@ import (
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/timing"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
@@ -179,6 +181,13 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
}
func (v *YoutubeVideo) download() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("download").Add(time.Since(start))
}(start)
if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
return errors.Err("video is a live stream and hasn't completed yet")
}
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777)
@@ -204,6 +213,7 @@ func (v *YoutubeVideo) download() error {
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
"--rm-cache-dir",
"--postprocessor-args",
"-movflags faststart",
"--abort-on-unavailable-fragment",
@@ -366,6 +376,10 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
}
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("publish").Add(time.Since(start))
}(start)
languages, locations, tags := v.getMetadata()
var fee *jsonrpc.Fee
if params.Fee != nil {
@@ -439,6 +453,8 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
return nil, errors.Err(err)
}
if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
return nil, errors.Err("video is too long to process")
}
for {
@@ -573,10 +589,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
v.walletLock.RLock()
defer v.walletLock.RUnlock()
if v.mocked {
start := time.Now()
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: streamCreateOptions,
FileSize: &videoSize,
})
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
if err != nil {
return nil, err
}
@@ -596,6 +614,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
start := time.Now()
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
ClearLanguages: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
@@ -603,6 +622,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
StreamCreateOptions: streamCreateOptions,
FileSize: &videoSize,
})
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
if err != nil {
return nil, err
}
+114
View File
@@ -0,0 +1,114 @@
package timing
import (
"sync"
"sync/atomic"
"time"
"github.com/lbryio/ytsync/metrics"
"github.com/sirupsen/logrus"
)
type Timing struct {
component string
milliseconds int64
min int64
max int64
invocations int32
}
var timings *sync.Map
func TimedComponent(component string) *Timing {
if timings == nil {
timings = &sync.Map{}
}
stored, _ := timings.LoadOrStore(component, &Timing{
component: component,
milliseconds: 0,
min: int64(99999999),
})
t, _ := stored.(*Timing)
return t
}
func ClearTimings() {
if timings == nil {
return
}
timings.Range(func(key interface{}, value interface{}) bool {
timings.Delete(key)
return true
})
}
func Report() {
var totalTime time.Duration
timings.Range(func(key interface{}, value interface{}) bool {
totalTime += value.(*Timing).Get()
return true
})
timings.Range(func(key interface{}, value interface{}) bool {
component := key
componentRuntime := value.(*Timing).Get().String()
percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
invocations := value.(*Timing).Invocations()
avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
minRuntime := value.(*Timing).Min().String()
maxRuntime := value.(*Timing).Max().String()
logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
component,
componentRuntime,
percentTime,
invocations,
avgTime,
minRuntime,
maxRuntime,
)
return true
})
}
func (t *Timing) Add(d time.Duration) {
metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
atomic.AddInt64(&t.milliseconds, d.Milliseconds())
for {
oldMin := atomic.LoadInt64(&t.min)
if d.Milliseconds() < oldMin {
if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
break
}
} else {
break
}
}
for {
oldMax := atomic.LoadInt64(&t.max)
if d.Milliseconds() > oldMax {
if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
break
}
} else {
break
}
}
atomic.AddInt32(&t.invocations, 1)
}
func (t *Timing) Get() time.Duration {
ms := atomic.LoadInt64(&t.milliseconds)
return time.Duration(ms) * time.Millisecond
}
func (t *Timing) Invocations() int32 {
return atomic.LoadInt32(&t.invocations)
}
func (t *Timing) Min() time.Duration {
ms := atomic.LoadInt64(&t.min)
return time.Duration(ms) * time.Millisecond
}
func (t *Timing) Max() time.Duration {
ms := atomic.LoadInt64(&t.max)
return time.Duration(ms) * time.Millisecond
}
+27 -1
View File
@@ -7,9 +7,11 @@ import (
"os/user"
"path/filepath"
"strconv"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/ytsync/timing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
@@ -248,9 +250,12 @@ func CleanupLbrynet() error {
db, err := os.Stat(lbryumDir + "/blockchain.db")
if err != nil {
if os.IsNotExist(err) {
return nil
}
return errors.Err(err)
}
dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
if db.Size() > dbSizeLimit {
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
if err != nil {
@@ -266,7 +271,24 @@ func CleanupLbrynet() error {
return nil
}
func SleepUntilQuotaReset() {
PST, _ := time.LoadLocation("America/Los_Angeles")
t := time.Now().In(PST)
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
d := n.Sub(t)
if d < 0 {
n = n.Add(24 * time.Hour)
d = n.Sub(t)
}
log.Infof("gotta sleep %s until the quota resets", d.String())
time.Sleep(d)
}
func StartDaemon() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("startDaemon").Add(time.Since(start))
}(start)
if IsUsingDocker() {
return startDaemonViaDocker()
}
@@ -274,6 +296,10 @@ func StartDaemon() error {
}
func StopDaemon() error {
start := time.Now()
defer func(start time.Time) {
timing.TimedComponent("stopDaemon").Add(time.Since(start))
}(start)
if IsUsingDocker() {
return stopDaemonViaDocker()
}