Compare commits

..
1 Commits
Author SHA1 Message Date
Alex Grintsvayg 8b32381b23 grin 2019-10-16 15:00:11 -04:00
16 changed files with 163 additions and 187 deletions

No files matched your search

+5 -2
View File
@@ -1,9 +1,12 @@
#blockchain_name: lbrycrd_main
#blockchain_name: lbrycrd_testnet
blockchain_name: lbrycrd_regtest
lbryum_servers:
- walletserver:50001
reflect_streams: false
# - spv1.lbry.com:50001 #Production Wallet Server
- walletserver:50001
save_blobs: true
save_files: false
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
share_usage_data: false
tcp_port: 3333
udp_port: 4444
+3 -3
View File
@@ -4,7 +4,7 @@ services:
## Lbrycrd ##
#############
lbrycrd:
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
image: lbry/lbrycrd:v0.17.2.1
restart: "no"
ports:
- "15201:29246"
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.48.1
image: lbry/wallet-server:v0.42.2
restart: always
environment:
- DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.48.1
image: lbry/lbrynet:v0.42.2
restart: always
ports:
- "15100:5279"
+13 -15
View File
@@ -11,9 +11,9 @@ function override_config_option() {
local var=$1 option=$2 config=$3
if [[ -v $var ]]; then
# Remove the existing config option:
sed -i "/^$option\W*=/d" "$config"
sed -i "/^$option\W*=/d" $config
# Add the value from the environment:
echo "$option=${!var}" >> "$config"
echo "$option=${!var}" >> $config
fi
}
@@ -38,17 +38,15 @@ function set_config() {
else
echo "Creating a fresh config file from environment variables."
## Set config params
{
echo "port=${PORT=9246}"
echo "rpcuser=${RPC_USER=lbry}"
echo "rpcpassword=${RPC_PASSWORD=lbry}"
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
echo "rpcport=${RPC_PORT=9245}"
echo "rpcbind=${RPC_BIND=0.0.0.0}"
echo "deprecatedrpc=accounts"
echo "deprecatedrpc=validateaddress"
echo "deprecatedrpc=signrawtransaction"
} >> $CONFIG_PATH
echo "port=${PORT=9246}" > $CONFIG_PATH
echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
fi
echo "Config: "
cat $CONFIG_PATH
@@ -81,7 +79,7 @@ case $RUN_MODE in
regtest )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p "$(dirname $CONFIG_PATH)"
mkdir -p `dirname $CONFIG_PATH`
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
@@ -103,7 +101,7 @@ case $RUN_MODE in
testnet )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p "$(dirname $CONFIG_PATH)"
mkdir -p `dirname $CONFIG_PATH`
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
+1 -1
View File
@@ -8,7 +8,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.48.1
image: lbry/lbrynet:v0.38.6
restart: "no"
networks:
lbry-network:
+1 -1
View File
@@ -18,7 +18,7 @@ require (
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
+2 -2
View File
@@ -213,8 +213,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e h1:B1ekb7lXFuMol/tUdwTajOe56ab38drBJmr0X8IoZYE=
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e/go.mod h1:Fm4jBlDyDauHIBC5OABuqcBWKBpcbRgPk5K0eG2mhSs=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
@@ -1,4 +1,4 @@
package ip_manager
package ipManager
import (
"github.com/asaskevich/govalidator"
@@ -13,7 +13,7 @@ import (
"time"
)
const IPCooldownPeriod = 35 * time.Second
const IPCooldownPeriod = 25 * time.Second
const unbanTimeout = 3 * time.Hour
var ipv6Pool []string
@@ -37,7 +37,7 @@ func GetNextIP(ipv6 bool) (string, error) {
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
+1
View File
@@ -92,6 +92,7 @@ const (
TransferStateNotTouched = iota
TransferStatePending
TransferStateComplete
TransferStateFailed = -1
)
func (s *SyncManager) Start() error {
+29 -52
View File
@@ -10,10 +10,9 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/util"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/tags_manager"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
logUtils "github.com/lbryio/ytsync/util"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
@@ -22,21 +21,14 @@ import (
)
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList(1, 50)
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
ledger := "lbc_mainnet"
accounts := accountsResponse.LBCMainnet
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
accounts = accountsResponse.LBCRegtest
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
accounts = append(accounts, a)
}
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: util.PtrToInt(1000),
@@ -128,13 +120,13 @@ func (s *Sync) walletSetup() error {
}
}
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
claimAddress, err := s.daemon.AddressList(nil, nil)
if err != nil {
return err
} else if claimAddress == nil {
return errors.Err("could not get an address")
return errors.Err("could not get unused address")
}
s.claimAddress = string(claimAddress.Items[0].Address)
s.claimAddress = string((*claimAddress)[0].Address)
if s.claimAddress == "" {
return errors.Err("found blank claim address")
}
@@ -152,23 +144,20 @@ func (s *Sync) walletSetup() error {
func (s *Sync) getDefaultAccount() (string, error) {
if s.defaultAccountID == "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
accounts, err := s.daemon.AccountList()
if err != nil {
return "", errors.Err(err)
}
ledger := "lbc_mainnet"
accountsNet := (*accounts).LBCMainnet
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
accountsNet = (*accounts).LBCRegtest
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.IsDefault {
s.defaultAccountID = a.ID
break
}
for _, account := range accountsNet {
if account.IsDefault {
s.defaultAccountID = account.ID
break
}
}
if s.defaultAccountID == "" {
return "", errors.Err("No default account found")
}
@@ -182,7 +171,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
return err
}
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
utxolist, err := s.daemon.UTXOList(&defaultAccount)
if err != nil {
return err
} else if utxolist == nil {
@@ -194,7 +183,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
count := 0
confirmedCount := 0
for _, utxo := range utxolist.Items {
for _, utxo := range *utxolist {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
@@ -250,11 +239,21 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
func (s *Sync) waitForNewBlock() error {
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
}
status, err := s.daemon.Status()
if err != nil {
return err
}
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
time.Sleep(5 * time.Second)
status, err = s.daemon.Status()
@@ -262,18 +261,11 @@ func (s *Sync) waitForNewBlock() error {
return err
}
}
currentBlock := status.Wallet.Blocks
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
if i%3 == 0 {
log.Printf("Waiting for new block (%d)...", currentBlock+1)
}
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
err = s.GenerateRegtestBlock()
if err != nil {
return err
}
}
time.Sleep(10 * time.Second)
status, err = s.daemon.Status()
if err != nil {
@@ -283,21 +275,6 @@ func (s *Sync) waitForNewBlock() error {
return nil
}
func (s *Sync) GenerateRegtestBlock() error {
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block: ", err)
}
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
return nil
}
func (s *Sync) ensureChannelOwnership() error {
if s.LbryChannelName == "" {
return errors.Err("no channel name set")
@@ -409,7 +386,7 @@ func (s *Sync) ensureChannelOwnership() error {
claimCreateOptions := jsonrpc.ClaimCreateOptions{
Title: &channelInfo.Title,
Description: &channelInfo.Description,
Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
+62 -66
View File
@@ -6,6 +6,8 @@ import (
"strings"
"time"
"github.com/uber-go/atomic"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
@@ -24,14 +26,14 @@ func waitConfirmations(s *Sync) error {
allConfirmed := false
waiting:
for !allConfirmed {
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
utxolist, err := s.daemon.UTXOList(&defaultAccount)
if err != nil {
return err
} else if utxolist == nil {
return errors.Err("no response")
}
for _, utxo := range utxolist.Items {
for _, utxo := range *utxolist {
if utxo.Confirmations <= 0 {
err = s.waitForNewBlock()
if err != nil {
@@ -45,13 +47,13 @@ waiting:
return nil
}
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
func abandonSupports(s *Sync) (float64, error) {
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
defaultAccount, err := s.getDefaultAccount()
@@ -66,23 +68,25 @@ func abandonSupports(s *Sync) (float64, error) {
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
}
producerWG := &stop.Group{}
claimIDChan := make(chan string, len(allSupports))
abandonRspChan := make(chan abandonResponse, len(allSupports))
claimIDChan := make(chan string)
abandonRspChan := make(chan abandonResponse)
collectorChan := make(chan bool)
alreadyAbandoned := make(map[string]bool, len(allSupports))
producerWG.Add(1)
totalAbandoned := atomic.NewFloat64(0)
go func() {
defer producerWG.Done()
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
continue
}
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
totalAbandoned.Add(r.Amount)
}
close(collectorChan)
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
@@ -95,72 +99,64 @@ func abandonSupports(s *Sync) (float64, error) {
claimID, more := <-claimIDChan
if !more {
return
} else {
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
} else {
}
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
Amount: 0,
}
} else {
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: err}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: nil,
Amount: outputAmount,
Error: errors.Err("abandoning supports: no outputs for %s", claimID),
}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: errors.Err(err)}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
abandonRspChan <- abandonResponse{ClaimID: claimID, Amount: outputAmount}
continue
}
}()
}
producerWG.Wait()
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
totalAbandoned := 0.0
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
continue
}
totalAbandoned += r.Amount
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
}
return totalAbandoned, nil
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
<-collectorChan
return totalAbandoned.Load(), nil
}
type updateInfo struct {
@@ -177,7 +173,7 @@ func transferVideos(s *Sync) error {
if err != nil {
return err
}
streams, err := s.daemon.StreamList(&account, 1, 30000)
streams, err := s.daemon.StreamList(&account)
if err != nil {
return errors.Err(err)
}
@@ -191,7 +187,7 @@ func transferVideos(s *Sync) error {
}
var stream *jsonrpc.Claim = nil
for _, c := range streams.Items {
for _, c := range *streams {
if c.ClaimID != video.ClaimID {
continue
}
+18 -18
View File
@@ -333,7 +333,7 @@ func (s *Sync) processTransfers() (e error) {
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
}
if supportAmount > 0 {
logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
logUtils.SendInfoToSlack("%.6f LBCs were abandoned and should be used as support", supportAmount)
}
err = transferVideos(s)
if err != nil {
@@ -354,7 +354,7 @@ func (s *Sync) processTransfers() (e error) {
return err
}
isTip := true
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
if err != nil {
return errors.Err(err)
}
@@ -384,7 +384,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
var transferState *int
if s.shouldTransfer() {
if *e == nil {
if *e != nil {
transferState = util.PtrToInt(TransferStateFailed)
} else {
transferState = util.PtrToInt(TransferStateComplete)
}
}
@@ -670,12 +672,12 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
}
account = &a
}
claims, err := s.daemon.StreamList(account, 1, 30000)
claims, err := s.daemon.StreamList(account)
if err != nil {
return nil, errors.Prefix("cannot list claims", err)
}
return claims.Items, nil
return *claims, nil
}
func (s *Sync) checkIntegrity() error {
@@ -822,9 +824,9 @@ func (s *Sync) startWorker(workerNum int) {
err := s.processVideo(v)
if err != nil {
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), errors.FullTrace(err))
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
if errors.Is(err, sources.ErrInterruptedByUser) {
return
}
fatalErrors := []string{
@@ -1134,19 +1136,17 @@ func (s *Sync) processVideo(v video) (err error) {
func (s *Sync) importPublicKey() error {
if s.publicKey != "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
ledger := "lbc_mainnet"
accounts := accountsResponse.LBCMainnet
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
accounts = accountsResponse.LBCRegtest
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.PublicKey == s.publicKey {
return nil
}
for _, a := range accounts {
if a.PublicKey == s.publicKey {
return nil
}
}
log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
@@ -1174,12 +1174,12 @@ func (s *Sync) getUnsentSupports() (float64, error) {
if err != nil {
return 0, errors.Err(err)
}
transactionList, err := s.daemon.TransactionList(&defaultAccount, 1, 90000)
transactionList, err := s.daemon.TransactionList(&defaultAccount)
if err != nil {
return 0, errors.Err(err)
}
sentSupports := 0.0
for _, t := range transactionList.Items {
for _, t := range *transactionList {
if len(t.SupportInfo) == 0 {
continue
}
@@ -1194,7 +1194,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
}
}
if balanceAmount > 10 && sentSupports < 1 {
logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
logUtils.SendErrorToSlack("this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", balanceAmount, sentSupports)
return balanceAmount - 10, nil
}
}
+6 -6
View File
@@ -86,7 +86,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cp)
}
@@ -185,9 +185,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode >= http.StatusInternalServerError {
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
@@ -232,7 +232,7 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
@@ -271,7 +271,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
@@ -344,7 +344,7 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
+12 -10
View File
@@ -18,10 +18,10 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/ipManager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tags_manager"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
duration "github.com/ChannelMeter/iso8601duration"
@@ -31,6 +31,8 @@ import (
"google.golang.org/api/youtube/v3"
)
var ErrInterruptedByUser = errors.Base("interrupted by user")
type YoutubeVideo struct {
id string
title string
@@ -220,18 +222,18 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
)
}
sourceAddress, err := ip_manager.GetNextIP(useIPv6)
sourceAddress, err := ipManager.GetNextIP(useIPv6)
if err != nil {
if sourceAddress == "throttled" {
for {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
return errors.Err(ErrInterruptedByUser)
default:
}
time.Sleep(ip_manager.IPCooldownPeriod)
sourceAddress, err = ip_manager.GetNextIP(useIPv6)
time.Sleep(ipManager.IPCooldownPeriod)
sourceAddress, err = ipManager.GetNextIP(useIPv6)
if err == nil {
break
}
@@ -240,7 +242,7 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
return errors.Err(err)
}
}
defer ip_manager.ReleaseIP(sourceAddress)
defer ipManager.ReleaseIP(sourceAddress)
if useIPv6 {
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
@@ -282,7 +284,7 @@ runcmd:
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") {
ip_manager.SetIpThrottled(sourceAddress, v.stopGroup)
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++
goto runcmd
@@ -477,7 +479,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
tags = v.youtubeInfo.Snippet.Tags
}
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
if err != nil {
log.Errorln(err.Error())
}
@@ -489,7 +491,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
if err != nil {
return nil, errors.Err(err)
}
@@ -1,4 +1,4 @@
package tags_manager
package tagsManager
import (
"regexp"
@@ -1,4 +1,4 @@
package tags_manager
package tagsManager
import (
"fmt"
+5 -6
View File
@@ -61,12 +61,11 @@ func (u *thumbnailUploader) uploadThumbnail() error {
uploader := s3manager.NewUploader(s3Session)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
CacheControl: aws.String("public, max-age=2592000"),
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
})
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)