Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
749d5095c9 | ||
|
|
ba77b61ae0 | ||
|
|
07915ce697 | ||
|
|
4b2221ce5e | ||
|
|
1d5b69bfe6 | ||
|
|
4d915587e0 | ||
|
|
83f2c28c20 | ||
|
|
47e467057d | ||
|
|
1ae32d638b |
No files matched your search
+4
-6
@@ -2,8 +2,7 @@ os: linux
|
||||
dist: bionic
|
||||
language: go
|
||||
go:
|
||||
- 1.12.7
|
||||
- master
|
||||
- 1.14.4
|
||||
|
||||
install: true
|
||||
|
||||
@@ -21,15 +20,14 @@ addons:
|
||||
packages:
|
||||
- ffmpeg
|
||||
- tree
|
||||
- python-pip
|
||||
- python3-pip
|
||||
|
||||
before_script:
|
||||
- sudo pip install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
|
||||
- sudo pip3 install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
|
||||
|
||||
env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
var dbHandle *db.SQL
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
module github.com/lbryio/ytsync
|
||||
module github.com/lbryio/ytsync/v5
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/manager"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
ytUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
+9
-6
@@ -6,11 +6,11 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
@@ -215,6 +215,9 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "quotaExceeded") {
|
||||
logUtils.SleepUntilQuotaReset()
|
||||
}
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||
@@ -224,8 +227,8 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
}
|
||||
|
||||
+4
-4
@@ -10,11 +10,11 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
+4
-4
@@ -10,8 +10,8 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -95,8 +95,8 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}()
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(30 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(20 * time.Minute)
|
||||
s.daemon.SetRPCTimeout(60 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
|
||||
+8
-8
@@ -14,13 +14,13 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/sources"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
@@ -307,7 +307,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
log.Infoln("Waiting for daemon to finish starting...")
|
||||
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
|
||||
s.daemon.SetRPCTimeout(20 * time.Minute)
|
||||
s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
|
||||
err = s.waitForDaemonStart()
|
||||
if err != nil {
|
||||
|
||||
+8
-2
@@ -2,9 +2,11 @@ package metrics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -19,7 +21,11 @@ var (
|
||||
func getHostname() string {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = "ytsync-unknown"
|
||||
hostname = "ytsync_unknown"
|
||||
}
|
||||
return hostname
|
||||
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return reg.ReplaceAllString(hostname, "_")
|
||||
}
|
||||
+1
-1
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/null"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
)
|
||||
|
||||
type SyncSummary struct {
|
||||
|
||||
@@ -17,14 +17,14 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
|
||||
duration "github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
|
||||
+4
-1
@@ -5,7 +5,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/metrics"
|
||||
"github.com/lbryio/ytsync/v5/metrics"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -33,6 +33,9 @@ func TimedComponent(component string) *Timing {
|
||||
}
|
||||
|
||||
func ClearTimings() {
|
||||
if timings == nil {
|
||||
return
|
||||
}
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
timings.Delete(key)
|
||||
return true
|
||||
|
||||
+14
-1
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/ytsync/timing"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
@@ -271,6 +271,19 @@ func CleanupLbrynet() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SleepUntilQuotaReset() {
|
||||
PST, _ := time.LoadLocation("America/Los_Angeles")
|
||||
t := time.Now().In(PST)
|
||||
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
|
||||
d := n.Sub(t)
|
||||
if d < 0 {
|
||||
n = n.Add(24 * time.Hour)
|
||||
d = n.Sub(t)
|
||||
}
|
||||
log.Infof("gotta sleep %s until the quota resets", d.String())
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func StartDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
|
||||
Reference in new issue
Block a user