Compare commits

...
9 Commits
Author SHA1 Message Date
Niko Storni 749d5095c9 fix semver 2020-06-11 18:45:56 +02:00
Niko ba77b61ae0 Merge pull request #56 from lbryio/fix-travis
fix travis try 2
2020-06-09 21:48:40 -04:00
Niko Storni 07915ce697 fix travis try 2 2020-06-10 03:44:34 +02:00
Niko Storni 4b2221ce5e fix travis (try 1) 2020-06-10 03:38:42 +02:00
Niko Storni 1d5b69bfe6 add automatic sleep until quota reset
re-enable ipv6
2020-06-10 03:32:45 +02:00
Niko Storni 4d915587e0 temporarily disable ipv6 2020-05-30 02:18:42 +02:00
Niko Storni 83f2c28c20 fix nil ptr 2020-05-22 12:47:18 -04:00
Niko Storni 47e467057d change timeouts to avoid long wait times 2020-05-20 04:28:29 +02:00
Niko Storni 1ae32d638b fix prom allowed chars 2020-05-19 23:24:16 +02:00
16 changed files with 71 additions and 48 deletions

No files matched your search

+4 -6
View File
@@ -2,8 +2,7 @@ os: linux
dist: bionic
language: go
go:
- 1.12.7
- master
- 1.14.4
install: true
@@ -21,15 +20,14 @@ addons:
packages:
- ffmpeg
- tree
- python-pip
- python3-pip
before_script:
- sudo pip install -U youtube-dl
- sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
- sudo pip3 install -U youtube-dl
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/lbryio/reflector.go/store"
"github.com/sirupsen/logrus"
"github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/util"
)
var dbHandle *db.SQL
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"strconv"
"strings"
"github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/util"
"github.com/sirupsen/logrus"
)
+1 -1
View File
@@ -1,4 +1,4 @@
module github.com/lbryio/ytsync
module github.com/lbryio/ytsync/v5
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"github.com/asaskevich/govalidator"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/util"
log "github.com/sirupsen/logrus"
)
+3 -3
View File
@@ -9,9 +9,9 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/manager"
"github.com/lbryio/ytsync/sdk"
ytUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
+9 -6
View File
@@ -6,11 +6,11 @@ import (
"syscall"
"time"
"github.com/lbryio/ytsync/blobs_reflector"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/blobs_reflector"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
@@ -215,6 +215,9 @@ func (s *SyncManager) Start() error {
}
if err != nil {
if strings.Contains(err.Error(), "quotaExceeded") {
logUtils.SleepUntilQuotaReset()
}
fatalErrors := []string{
"default_wallet already exists",
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
@@ -224,8 +227,8 @@ func (s *SyncManager) Start() error {
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
"Daily Limit Exceeded",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
}
+4 -4
View File
@@ -10,11 +10,11 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/timing"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/ytsync/tags_manager"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
+4 -4
View File
@@ -10,8 +10,8 @@ import (
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/timing"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/timing"
log "github.com/sirupsen/logrus"
)
@@ -95,8 +95,8 @@ func abandonSupports(s *Sync) (float64, error) {
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
defer s.daemon.SetRPCTimeout(20 * time.Minute)
s.daemon.SetRPCTimeout(60 * time.Second)
defer s.daemon.SetRPCTimeout(5 * time.Minute)
for i := 0; i < s.ConcurrentVideos; i++ {
consumerWG.Add(1)
go func() {
+8 -8
View File
@@ -14,13 +14,13 @@ import (
"syscall"
"time"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/sources"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/ytsync/timing"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/sources"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
@@ -307,7 +307,7 @@ func (s *Sync) FullCycle() (e error) {
log.Infoln("Waiting for daemon to finish starting...")
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
s.daemon.SetRPCTimeout(20 * time.Minute)
s.daemon.SetRPCTimeout(5 * time.Minute)
err = s.waitForDaemonStart()
if err != nil {
+8 -2
View File
@@ -2,9 +2,11 @@ package metrics
import (
"os"
"regexp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
)
var (
@@ -19,7 +21,11 @@ var (
func getHostname() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "ytsync-unknown"
hostname = "ytsync_unknown"
}
return hostname
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err != nil {
log.Fatal(err)
}
return reg.ReplaceAllString(hostname, "_")
}
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/null"
"github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/util"
log "github.com/sirupsen/logrus"
)
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"sync"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/v5/namer"
)
type SyncSummary struct {
+7 -7
View File
@@ -17,14 +17,14 @@ import (
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/timing"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tags_manager"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
duration "github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
+4 -1
View File
@@ -5,7 +5,7 @@ import (
"sync/atomic"
"time"
"github.com/lbryio/ytsync/metrics"
"github.com/lbryio/ytsync/v5/metrics"
"github.com/sirupsen/logrus"
)
@@ -33,6 +33,9 @@ func TimedComponent(component string) *Timing {
}
func ClearTimings() {
if timings == nil {
return
}
timings.Range(func(key interface{}, value interface{}) bool {
timings.Delete(key)
return true
+14 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/ytsync/timing"
"github.com/lbryio/ytsync/v5/timing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
@@ -271,6 +271,19 @@ func CleanupLbrynet() error {
return nil
}
func SleepUntilQuotaReset() {
PST, _ := time.LoadLocation("America/Los_Angeles")
t := time.Now().In(PST)
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
d := n.Sub(t)
if d < 0 {
n = n.Add(24 * time.Hour)
d = n.Sub(t)
}
log.Infof("gotta sleep %s until the quota resets", d.String())
time.Sleep(d)
}
func StartDaemon() error {
start := time.Now()
defer func(start time.Time) {