Compare commits

...
4 Commits
Author SHA1 Message Date
Alex Grintsvayg 8b32381b23 grin 2019-10-16 15:00:11 -04:00
Niko Storni 291a105269 improve getDefaultAccount
use default account on more calls
improve API logging
fix memory leak
2019-10-16 19:38:45 +02:00
Thomas Zarebczan 682117a030 Merge pull request #45 from StrikerRUS/license
added license file
2019-10-15 11:52:59 -04:00
StrikerRUS a3b660fab5 aadded license file 2019-10-09 00:11:55 +03:00
7 changed files with 123 additions and 98 deletions

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
The MIT License (MIT)
Copyright (c) 2017-2019 LBRY Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+2 -1
View File
@@ -13,7 +13,6 @@ require (
github.com/docker/go-units v0.4.0 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
github.com/google/btree v1.0.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.3 // indirect
@@ -34,3 +33,5 @@ require (
google.golang.org/api v0.11.0
google.golang.org/appengine v1.6.5 // indirect
)
go 1.13
+19 -17
View File
@@ -141,26 +141,28 @@ func (s *Sync) walletSetup() error {
return nil
}
func (s *Sync) getDefaultAccount() (string, error) {
accounts, err := s.daemon.AccountList()
if err != nil {
return "", errors.Err(err)
}
accountsNet := (*accounts).LBCMainnet
if logUtils.IsRegTest() {
accountsNet = (*accounts).LBCRegtest
}
defaultAccount := ""
for _, account := range accountsNet {
if account.IsDefault {
defaultAccount = account.ID
break
if s.defaultAccountID == "" {
accounts, err := s.daemon.AccountList()
if err != nil {
return "", errors.Err(err)
}
accountsNet := (*accounts).LBCMainnet
if logUtils.IsRegTest() {
accountsNet = (*accounts).LBCRegtest
}
for _, account := range accountsNet {
if account.IsDefault {
s.defaultAccountID = account.ID
break
}
}
if s.defaultAccountID == "" {
return "", errors.Err("No default account found")
}
}
if defaultAccount == "" {
return "", errors.Err("No default account found")
}
return defaultAccount, nil
return s.defaultAccountID, nil
}
func (s *Sync) ensureEnoughUTXOs() error {
+58 -62
View File
@@ -6,6 +6,8 @@ import (
"strings"
"time"
"github.com/uber-go/atomic"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
@@ -45,13 +47,13 @@ waiting:
return nil
}
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
func abandonSupports(s *Sync) (float64, error) {
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
defaultAccount, err := s.getDefaultAccount()
@@ -66,23 +68,25 @@ func abandonSupports(s *Sync) (float64, error) {
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
}
producerWG := &stop.Group{}
claimIDChan := make(chan string, len(allSupports))
abandonRspChan := make(chan abandonResponse, len(allSupports))
claimIDChan := make(chan string)
abandonRspChan := make(chan abandonResponse)
collectorChan := make(chan bool)
alreadyAbandoned := make(map[string]bool, len(allSupports))
producerWG.Add(1)
totalAbandoned := atomic.NewFloat64(0)
go func() {
defer producerWG.Done()
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
continue
}
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
totalAbandoned.Add(r.Amount)
}
close(collectorChan)
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
@@ -95,72 +99,64 @@ func abandonSupports(s *Sync) (float64, error) {
claimID, more := <-claimIDChan
if !more {
return
} else {
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
} else {
}
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
Amount: 0,
}
} else {
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: err}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: nil,
Amount: outputAmount,
Error: errors.Err("abandoning supports: no outputs for %s", claimID),
}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: errors.Err(err)}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
abandonRspChan <- abandonResponse{ClaimID: claimID, Amount: outputAmount}
continue
}
}()
}
producerWG.Wait()
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
totalAbandoned := 0.0
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
continue
}
totalAbandoned += r.Amount
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
}
return totalAbandoned, nil
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
<-collectorChan
return totalAbandoned.Load(), nil
}
type updateInfo struct {
+8 -4
View File
@@ -89,6 +89,7 @@ type Sync struct {
transferState int
clientPublishAddress string
publicKey string
defaultAccountID string
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -342,7 +343,10 @@ func (s *Sync) processTransfers() (e error) {
if err != nil {
return err
}
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
reallocateSupports := supportAmount > 0.01
if reallocateSupports {
err = waitConfirmations(s)
@@ -350,7 +354,7 @@ func (s *Sync) processTransfers() (e error) {
return err
}
isTip := true
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
if err != nil {
return errors.Err(err)
}
@@ -820,9 +824,9 @@ func (s *Sync) startWorker(workerNum int) {
err := s.processVideo(v)
if err != nil {
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), errors.FullTrace(err))
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
if errors.Is(err, sources.ErrInterruptedByUser) {
return
}
fatalErrors := []string{
+18 -13
View File
@@ -82,13 +82,14 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cp)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiJobsResponse
err = json.Unmarshal(body, &response)
if err != nil {
@@ -139,14 +140,14 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiSetChannelCertResponse
err = json.Unmarshal(body, &response)
if err != nil {
@@ -182,13 +183,14 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
if err != nil {
return nil, nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiChannelStatusResponse
err = json.Unmarshal(body, &response)
if err != nil {
@@ -226,13 +228,14 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiChannelStatusResponse
err = json.Unmarshal(body, &response)
if err != nil {
@@ -264,13 +267,14 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
@@ -336,13 +340,14 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
+3 -1
View File
@@ -31,6 +31,8 @@ import (
"google.golang.org/api/youtube/v3"
)
var ErrInterruptedByUser = errors.Base("interrupted by user")
type YoutubeVideo struct {
id string
title string
@@ -226,7 +228,7 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
for {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
return errors.Err(ErrInterruptedByUser)
default:
}