Compare commits
27
Commits
v5.3.16
...
fix-travis
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
07915ce697 | ||
|
|
4b2221ce5e | ||
|
|
1d5b69bfe6 | ||
|
|
4d915587e0 | ||
|
|
83f2c28c20 | ||
|
|
47e467057d | ||
|
|
1ae32d638b | ||
|
|
7907ee3579 | ||
|
|
414ed1c130 | ||
|
|
3bb17b09d3 | ||
|
|
046f46267e | ||
|
|
adc3ffe194 | ||
|
|
76301d40ba | ||
|
|
963c450730 | ||
|
|
6e7bb994d0 | ||
|
|
95890cbabf | ||
|
|
69c2a91ec7 | ||
|
|
fbe09a692c | ||
|
|
0ccc6e7979 | ||
|
|
3eb74cc8b5 | ||
|
|
9bfb1656fe | ||
|
|
5d40d24804 | ||
|
|
79d3aadda8 | ||
|
|
eba59a0806 | ||
|
|
3fd4909cd1 | ||
|
|
b59372c1d8 | ||
|
|
c9e713e9df |
No files matched your search
+4
-6
@@ -2,8 +2,7 @@ os: linux
|
||||
dist: bionic
|
||||
language: go
|
||||
go:
|
||||
- 1.12.7
|
||||
- master
|
||||
- 1.14.4
|
||||
|
||||
install: true
|
||||
|
||||
@@ -21,15 +20,14 @@ addons:
|
||||
packages:
|
||||
- ffmpeg
|
||||
- tree
|
||||
- python-pip
|
||||
- python3-pip
|
||||
|
||||
before_script:
|
||||
- sudo pip install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
|
||||
- sudo pip3 install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
|
||||
|
||||
env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
|
||||
|
||||
+2
-2
@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
|
||||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0,1,1)"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.60.0
|
||||
image: lbry/wallet-server:v0.73.1
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.60.0
|
||||
image: lbry/lbrynet:v0.73.1
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.51.2
|
||||
image: lbry/lbrynet:v0.73.1
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.38.6
|
||||
image: lbry/wallet-server:v0.73.1
|
||||
restart: always
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -20,12 +20,13 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88
|
||||
github.com/lbryio/lbry.go/v2 v2.6.0
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/cobra v0.0.5
|
||||
|
||||
@@ -29,6 +29,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
|
||||
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
@@ -188,8 +189,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88 h1:SOn28lm4Bd8RY63hFVyt0d6SAD9OtFMEJVaKrO+pc/E=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.0 h1:qC2NsSZH2AeNDOP0Dw62cKNqHPy1eEDladmgnhOrIQc=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.0/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
@@ -208,6 +209,7 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCW
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
|
||||
@@ -248,9 +250,13 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
|
||||
@@ -43,15 +43,13 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
ipv6Added := false
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
@@ -130,7 +128,6 @@ func (i *IPPool) ReleaseAll() {
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"github.com/lbryio/ytsync/manager"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
ytUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -38,7 +40,10 @@ var (
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
}()
|
||||
cmd := &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
|
||||
+8
-2
@@ -135,6 +135,7 @@ func (s *SyncManager) Start() error {
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
MaxVideoLength: s.maxVideoLength,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
@@ -168,8 +169,9 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
maxVideoLength := s.maxVideoLength
|
||||
if c.TotalSubscribers < 1000 {
|
||||
s.maxVideoLength = 1.0
|
||||
maxVideoLength = 1.0
|
||||
}
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
@@ -178,6 +180,7 @@ func (s *SyncManager) Start() error {
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
MaxVideoLength: maxVideoLength,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
@@ -212,6 +215,9 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "quotaExceeded") {
|
||||
logUtils.SleepUntilQuotaReset()
|
||||
}
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||
@@ -221,8 +227,8 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
}
|
||||
|
||||
+40
-10
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
@@ -49,6 +50,10 @@ func (s *Sync) enableAddressReuse() error {
|
||||
return nil
|
||||
}
|
||||
func (s *Sync) walletSetup() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("walletSetup").Add(time.Since(start))
|
||||
}(start)
|
||||
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
|
||||
s.walletMux.Lock()
|
||||
defer s.walletMux.Unlock()
|
||||
@@ -151,6 +156,10 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
|
||||
}(start)
|
||||
if s.defaultAccountID == "" {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
if err != nil {
|
||||
@@ -177,6 +186,10 @@ func (s *Sync) getDefaultAccount() (string, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -196,7 +209,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
@@ -253,6 +266,10 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
|
||||
func (s *Sync) waitForNewBlock() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
|
||||
}(start)
|
||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||
status, err := s.daemon.Status()
|
||||
if err != nil {
|
||||
@@ -302,6 +319,10 @@ func (s *Sync) GenerateRegtestBlock() error {
|
||||
}
|
||||
|
||||
func (s *Sync) ensureChannelOwnership() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
|
||||
}(start)
|
||||
if s.LbryChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
@@ -422,15 +443,20 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
if s.transferState <= 1 {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
@@ -446,6 +472,10 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("addCredits").Add(time.Since(start))
|
||||
}(start)
|
||||
log.Printf("Adding %f credits", amountToAdd)
|
||||
lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
|
||||
if err != nil {
|
||||
|
||||
+41
-8
@@ -10,20 +10,25 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitConfirmations").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allConfirmed := false
|
||||
waitCount := 0
|
||||
waiting:
|
||||
for !allConfirmed {
|
||||
for !allConfirmed && waitCount < 2 {
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -37,6 +42,7 @@ waiting:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
waitCount++
|
||||
continue waiting
|
||||
}
|
||||
}
|
||||
@@ -52,6 +58,10 @@ type abandonResponse struct {
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("abandonSupports").Add(time.Since(start))
|
||||
}(start)
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
@@ -85,22 +95,23 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}()
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(30 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(40 * time.Minute)
|
||||
s.daemon.SetRPCTimeout(60 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
outer:
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
@@ -119,7 +130,7 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
if summary == nil || len(*summary) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
@@ -127,7 +138,19 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
var outputAmount float64
|
||||
for _, tx := range *summary {
|
||||
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue outer
|
||||
}
|
||||
outputAmount += amount
|
||||
}
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
@@ -170,6 +193,10 @@ type updateInfo struct {
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferVideos").Add(time.Since(start))
|
||||
}(start)
|
||||
cleanTransfer := true
|
||||
|
||||
streamChan := make(chan updateInfo, s.ConcurrentVideos)
|
||||
@@ -258,7 +285,9 @@ func transferVideos(s *Sync) error {
|
||||
}
|
||||
|
||||
func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
start := time.Now()
|
||||
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
|
||||
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
|
||||
if updateError != nil {
|
||||
ui.videoStatus.FailureReason = updateError.Error()
|
||||
ui.videoStatus.Status = VideoStatusTranferFailed
|
||||
@@ -275,6 +304,10 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
}
|
||||
|
||||
func transferChannel(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferChannel").Add(time.Since(start))
|
||||
}(start)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+36
-9
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
@@ -91,6 +92,7 @@ type Sync struct {
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
defaultAccountID string
|
||||
MaxVideoLength float64
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -241,7 +243,7 @@ func (s *Sync) setStatusSyncing() error {
|
||||
|
||||
func (s *Sync) setExceptions() {
|
||||
if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
|
||||
s.Manager.maxVideoLength = 0.0 // skips max length checks
|
||||
s.MaxVideoLength = 9999.0 // skips max length checks
|
||||
s.Manager.maxVideoSize = 0
|
||||
}
|
||||
}
|
||||
@@ -257,7 +259,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
}
|
||||
|
||||
s.setExceptions()
|
||||
|
||||
defer timing.ClearTimings()
|
||||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stopGroup
|
||||
@@ -276,7 +278,6 @@ func (s *Sync) FullCycle() (e error) {
|
||||
}
|
||||
|
||||
defer s.setChannelTerminationStatus(&e)
|
||||
|
||||
err = s.downloadWallet()
|
||||
if err != nil && err.Error() != "wallet not on S3" {
|
||||
return errors.Prefix("failure in downloading wallet", err)
|
||||
@@ -306,7 +307,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
log.Infoln("Waiting for daemon to finish starting...")
|
||||
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
|
||||
s.daemon.SetRPCTimeout(40 * time.Minute)
|
||||
s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
|
||||
err = s.waitForDaemonStart()
|
||||
if err != nil {
|
||||
@@ -319,10 +320,10 @@ func (s *Sync) FullCycle() (e error) {
|
||||
}
|
||||
|
||||
if s.shouldTransfer() {
|
||||
return s.processTransfers()
|
||||
err = s.processTransfers()
|
||||
}
|
||||
|
||||
return nil
|
||||
timing.Report()
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Sync) processTransfers() (e error) {
|
||||
@@ -427,6 +428,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
|
||||
func (s *Sync) waitForDaemonStart() error {
|
||||
beginTime := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitForDaemonStart").Add(time.Since(start))
|
||||
}(beginTime)
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
@@ -504,6 +508,10 @@ func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
|
||||
// fixDupes abandons duplicate claims
|
||||
func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("fixDupes").Add(time.Since(start))
|
||||
}(start)
|
||||
abandonedClaims := false
|
||||
videoIDs := make(map[string]jsonrpc.Claim)
|
||||
for _, c := range claims {
|
||||
@@ -713,6 +721,10 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("checkIntegrity").Add(time.Since(start))
|
||||
}(start)
|
||||
allClaims, err := s.getClaims(false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -781,6 +793,10 @@ func (s *Sync) doSync() error {
|
||||
if err != nil {
|
||||
return errors.Prefix("could not import the transferee public key", err)
|
||||
}
|
||||
_, err = s.daemon.UTXORelease(nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("could not run uxo_release", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
@@ -899,6 +915,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -975,7 +992,13 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
}
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string
|
||||
|
||||
func (s *Sync) enqueueYoutubeVideos() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
|
||||
}(start)
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
@@ -1029,7 +1052,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
if youtubeIsLying {
|
||||
break
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
if s.YoutubeChannelID == mostRecentlyFailedChannel {
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = s.YoutubeChannelID
|
||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||
}
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
@@ -1154,7 +1181,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
ChannelID: s.lbryChannelID,
|
||||
MaxVideoSize: s.Manager.maxVideoSize,
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
MaxVideoLength: s.MaxVideoLength,
|
||||
Fee: s.Fee,
|
||||
DefaultAccount: da,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "ytsync",
|
||||
Subsystem: getHostname(),
|
||||
Name: "duration",
|
||||
Help: "The durations of the individual modules",
|
||||
}, []string{"path"})
|
||||
)
|
||||
|
||||
func getHostname() string {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = "ytsync_unknown"
|
||||
}
|
||||
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return reg.ReplaceAllString(hostname, "_")
|
||||
}
|
||||
+1
-1
@@ -146,7 +146,7 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
@@ -179,6 +181,13 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -204,6 +213,7 @@ func (v *YoutubeVideo) download() error {
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--rm-cache-dir",
|
||||
"--postprocessor-args",
|
||||
"-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
@@ -366,6 +376,10 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("publish").Add(time.Since(start))
|
||||
}(start)
|
||||
languages, locations, tags := v.getMetadata()
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
@@ -439,6 +453,8 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
|
||||
log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
for {
|
||||
@@ -573,10 +589,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
v.walletLock.RLock()
|
||||
defer v.walletLock.RUnlock()
|
||||
if v.mocked {
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -596,6 +614,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
|
||||
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
@@ -603,6 +622,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package timing
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/metrics"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Timing struct {
|
||||
component string
|
||||
milliseconds int64
|
||||
min int64
|
||||
max int64
|
||||
invocations int32
|
||||
}
|
||||
|
||||
var timings *sync.Map
|
||||
|
||||
func TimedComponent(component string) *Timing {
|
||||
if timings == nil {
|
||||
timings = &sync.Map{}
|
||||
}
|
||||
stored, _ := timings.LoadOrStore(component, &Timing{
|
||||
component: component,
|
||||
milliseconds: 0,
|
||||
min: int64(99999999),
|
||||
})
|
||||
t, _ := stored.(*Timing)
|
||||
return t
|
||||
}
|
||||
|
||||
func ClearTimings() {
|
||||
if timings == nil {
|
||||
return
|
||||
}
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
timings.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func Report() {
|
||||
var totalTime time.Duration
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
totalTime += value.(*Timing).Get()
|
||||
return true
|
||||
})
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
component := key
|
||||
componentRuntime := value.(*Timing).Get().String()
|
||||
percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
|
||||
invocations := value.(*Timing).Invocations()
|
||||
avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
|
||||
minRuntime := value.(*Timing).Min().String()
|
||||
maxRuntime := value.(*Timing).Max().String()
|
||||
logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
|
||||
component,
|
||||
componentRuntime,
|
||||
percentTime,
|
||||
invocations,
|
||||
avgTime,
|
||||
minRuntime,
|
||||
maxRuntime,
|
||||
)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Timing) Add(d time.Duration) {
|
||||
metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
|
||||
atomic.AddInt64(&t.milliseconds, d.Milliseconds())
|
||||
for {
|
||||
oldMin := atomic.LoadInt64(&t.min)
|
||||
if d.Milliseconds() < oldMin {
|
||||
if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
for {
|
||||
oldMax := atomic.LoadInt64(&t.max)
|
||||
if d.Milliseconds() > oldMax {
|
||||
if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
atomic.AddInt32(&t.invocations, 1)
|
||||
}
|
||||
|
||||
func (t *Timing) Get() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.milliseconds)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
|
||||
func (t *Timing) Invocations() int32 {
|
||||
return atomic.LoadInt32(&t.invocations)
|
||||
}
|
||||
|
||||
func (t *Timing) Min() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.min)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
func (t *Timing) Max() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.max)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
+24
-1
@@ -7,9 +7,11 @@ import (
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
@@ -253,7 +255,7 @@ func CleanupLbrynet() error {
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
|
||||
dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
|
||||
if db.Size() > dbSizeLimit {
|
||||
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
|
||||
if err != nil {
|
||||
@@ -269,7 +271,24 @@ func CleanupLbrynet() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SleepUntilQuotaReset() {
|
||||
PST, _ := time.LoadLocation("America/Los_Angeles")
|
||||
t := time.Now().In(PST)
|
||||
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
|
||||
d := n.Sub(t)
|
||||
if d < 0 {
|
||||
n = n.Add(24 * time.Hour)
|
||||
d = n.Sub(t)
|
||||
}
|
||||
log.Infof("gotta sleep %s until the quota resets", d.String())
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func StartDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("startDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return startDaemonViaDocker()
|
||||
}
|
||||
@@ -277,6 +296,10 @@ func StartDaemon() error {
|
||||
}
|
||||
|
||||
func StopDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("stopDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return stopDaemonViaDocker()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user