Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
963c450730 | ||
|
|
6e7bb994d0 | ||
|
|
95890cbabf | ||
|
|
69c2a91ec7 | ||
|
|
fbe09a692c | ||
|
|
0ccc6e7979 | ||
|
|
3eb74cc8b5 | ||
|
|
9bfb1656fe | ||
|
|
5d40d24804 | ||
|
|
79d3aadda8 | ||
|
|
eba59a0806 | ||
|
|
3fd4909cd1 | ||
|
|
b59372c1d8 | ||
|
|
c9e713e9df | ||
|
|
bc72d2f1c4 | ||
|
|
df539e5d01 | ||
|
|
1b89104101 | ||
|
|
3dc30c3b98 | ||
|
|
259df8d257 | ||
|
|
5460224cd6 |
No files matched your search
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.51.2
|
||||
image: lbry/wallet-server:v0.67.0
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.51.2
|
||||
image: lbry/lbrynet:v0.67.0
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
|
||||
@@ -20,7 +20,7 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88
|
||||
github.com/lbryio/lbry.go/v2 v2.5.3
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
|
||||
@@ -188,8 +188,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88 h1:SOn28lm4Bd8RY63hFVyt0d6SAD9OtFMEJVaKrO+pc/E=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.7-0.20200204053243-1a30fb743b88/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
|
||||
github.com/lbryio/lbry.go/v2 v2.5.3 h1:Qw4APDuVZDkwjQ49usy4p/ty/KfuQr18ld6GiREYlog=
|
||||
github.com/lbryio/lbry.go/v2 v2.5.3/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
|
||||
@@ -43,15 +43,13 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
ipv6Added := false
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
@@ -130,7 +128,6 @@ func (i *IPPool) ReleaseAll() {
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -135,6 +135,7 @@ func (s *SyncManager) Start() error {
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
MaxVideoLength: s.maxVideoLength,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
@@ -168,6 +169,10 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
maxVideoLength := s.maxVideoLength
|
||||
if c.TotalSubscribers < 1000 {
|
||||
maxVideoLength = 1.0
|
||||
}
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
@@ -175,6 +180,7 @@ func (s *SyncManager) Start() error {
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
MaxVideoLength: maxVideoLength,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
|
||||
+18
-10
@@ -196,7 +196,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
@@ -222,6 +222,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
if desiredUTXOCount < uint64(confirmedCount) {
|
||||
return nil
|
||||
}
|
||||
availableBalance, _ := balance.Available.Float64()
|
||||
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
|
||||
|
||||
@@ -419,15 +422,20 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
if s.transferState <= 1 {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
|
||||
+18
-5
@@ -86,21 +86,22 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(30 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(40 * time.Minute)
|
||||
defer s.daemon.SetRPCTimeout(20 * time.Minute)
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
outer:
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
@@ -119,7 +120,7 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
if summary == nil || len(*summary) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
@@ -127,7 +128,19 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
var outputAmount float64
|
||||
for _, tx := range *summary {
|
||||
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue outer
|
||||
}
|
||||
outputAmount += amount
|
||||
}
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
|
||||
+16
-4
@@ -91,6 +91,7 @@ type Sync struct {
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
defaultAccountID string
|
||||
MaxVideoLength float64
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -241,7 +242,7 @@ func (s *Sync) setStatusSyncing() error {
|
||||
|
||||
func (s *Sync) setExceptions() {
|
||||
if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
|
||||
s.Manager.maxVideoLength = 0.0 // skips max length checks
|
||||
s.MaxVideoLength = 9999.0 // skips max length checks
|
||||
s.Manager.maxVideoSize = 0
|
||||
}
|
||||
}
|
||||
@@ -306,7 +307,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
log.Infoln("Waiting for daemon to finish starting...")
|
||||
s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
|
||||
s.daemon.SetRPCTimeout(40 * time.Minute)
|
||||
s.daemon.SetRPCTimeout(20 * time.Minute)
|
||||
|
||||
err = s.waitForDaemonStart()
|
||||
if err != nil {
|
||||
@@ -781,6 +782,10 @@ func (s *Sync) doSync() error {
|
||||
if err != nil {
|
||||
return errors.Prefix("could not import the transferee public key", err)
|
||||
}
|
||||
_, err = s.daemon.UTXORelease(nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("could not run uxo_release", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
@@ -899,6 +904,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -975,6 +981,8 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
}
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string
|
||||
|
||||
func (s *Sync) enqueueYoutubeVideos() error {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
@@ -1029,7 +1037,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
if youtubeIsLying {
|
||||
break
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
if s.YoutubeChannelID == mostRecentlyFailedChannel {
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = s.YoutubeChannelID
|
||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||
}
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
@@ -1154,7 +1166,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
ChannelID: s.lbryChannelID,
|
||||
MaxVideoSize: s.Manager.maxVideoSize,
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
MaxVideoLength: s.MaxVideoLength,
|
||||
Fee: s.Fee,
|
||||
DefaultAccount: da,
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ type Fee struct {
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
TotalSubscribers uint `json:"total_subscribers"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
@@ -179,6 +180,9 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -204,6 +208,7 @@ func (v *YoutubeVideo) download() error {
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--rm-cache-dir",
|
||||
"--postprocessor-args",
|
||||
"-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
@@ -434,6 +439,15 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
var err error
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
|
||||
log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
|
||||
@@ -49,6 +49,19 @@ func GetLBRYNetDir() string {
|
||||
return lbrynetDir
|
||||
}
|
||||
|
||||
func GetLbryumDir() string {
|
||||
lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR")
|
||||
if lbryumDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbryum/"
|
||||
}
|
||||
return lbryumDir + "/"
|
||||
}
|
||||
|
||||
const ALL = true
|
||||
const ONLINE = false
|
||||
|
||||
@@ -225,6 +238,34 @@ func CleanupLbrynet() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbryumDir := GetLbryumDir()
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
lbryumDir = lbryumDir + ledger
|
||||
|
||||
db, err := os.Stat(lbryumDir + "/blockchain.db")
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
|
||||
if db.Size() > dbSizeLimit {
|
||||
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user