Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
77d0efcf3c | ||
|
|
291a105269 | ||
|
|
682117a030 | ||
|
|
6d2e4aa6e6 | ||
|
|
77749f05ba | ||
|
|
a3b660fab5 |
No files matched your search
@@ -0,0 +1,15 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017-2019 LBRY Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -1,4 +1,4 @@
|
||||
package ipManager
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
@@ -92,7 +92,6 @@ const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateFailed = -1
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
|
||||
+23
-20
@@ -10,10 +10,11 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
@@ -141,26 +142,28 @@ func (s *Sync) walletSetup() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
accounts, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
accountsNet := (*accounts).LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
accountsNet = (*accounts).LBCRegtest
|
||||
}
|
||||
defaultAccount := ""
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefault {
|
||||
defaultAccount = account.ID
|
||||
break
|
||||
if s.defaultAccountID == "" {
|
||||
accounts, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
accountsNet := (*accounts).LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
accountsNet = (*accounts).LBCRegtest
|
||||
}
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefault {
|
||||
s.defaultAccountID = account.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if s.defaultAccountID == "" {
|
||||
return "", errors.Err("No default account found")
|
||||
}
|
||||
}
|
||||
if defaultAccount == "" {
|
||||
return "", errors.Err("No default account found")
|
||||
}
|
||||
return defaultAccount, nil
|
||||
return s.defaultAccountID, nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
@@ -384,7 +387,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
|
||||
+12
-15
@@ -89,6 +89,7 @@ type Sync struct {
|
||||
transferState int
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
defaultAccountID string
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -342,7 +343,10 @@ func (s *Sync) processTransfers() (e error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reallocateSupports := supportAmount > 0.01
|
||||
if reallocateSupports {
|
||||
err = waitConfirmations(s)
|
||||
@@ -350,7 +354,7 @@ func (s *Sync) processTransfers() (e error) {
|
||||
return err
|
||||
}
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, nil)
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -380,9 +384,7 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
var transferState *int
|
||||
|
||||
if s.shouldTransfer() {
|
||||
if *e != nil {
|
||||
transferState = util.PtrToInt(TransferStateFailed)
|
||||
} else {
|
||||
if *e == nil {
|
||||
transferState = util.PtrToInt(TransferStateComplete)
|
||||
}
|
||||
}
|
||||
@@ -660,8 +662,6 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
}
|
||||
|
||||
func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
totalPages := uint64(1)
|
||||
var allClaims []jsonrpc.Claim
|
||||
var account *string = nil
|
||||
if defaultOnly {
|
||||
a, err := s.getDefaultAccount()
|
||||
@@ -670,15 +670,12 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
account = &a
|
||||
}
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
claims, err := s.daemon.ClaimList(account, page, 50)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
allClaims = append(allClaims, (*claims).Claims...)
|
||||
totalPages = (*claims).TotalPages
|
||||
claims, err := s.daemon.StreamList(account)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
return allClaims, nil
|
||||
|
||||
return *claims, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
|
||||
+59
-11
@@ -14,6 +14,8 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/null"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -82,6 +84,12 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cp)
|
||||
}
|
||||
var response apiJobsResponse
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
@@ -133,10 +141,15 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
var response apiSetChannelCertResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -145,7 +158,6 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) {
|
||||
@@ -167,11 +179,20 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
if transferState != nil {
|
||||
params.Add("transfer_state", strconv.Itoa(*transferState))
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, params)
|
||||
res, err := http.PostForm(endpoint, params)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
var response apiChannelStatusResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
@@ -199,15 +220,24 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
Data string `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/set_channel_claim_id"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"channel_id": {channelID},
|
||||
"auth_token": {a.ApiToken},
|
||||
"channel_claim_id": {channelClaimID},
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
var response apiChannelStatusResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -233,15 +263,24 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
"video_ids": {videoIDs},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -297,15 +336,24 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
|
||||
if status.IsTransferred != nil {
|
||||
vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -18,10 +18,10 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ipManager"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
duration "github.com/ChannelMeter/iso8601duration"
|
||||
@@ -220,7 +220,7 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
)
|
||||
}
|
||||
sourceAddress, err := ipManager.GetNextIP(useIPv6)
|
||||
sourceAddress, err := ip_manager.GetNextIP(useIPv6)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
@@ -230,8 +230,8 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
default:
|
||||
}
|
||||
|
||||
time.Sleep(ipManager.IPCooldownPeriod)
|
||||
sourceAddress, err = ipManager.GetNextIP(useIPv6)
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = ip_manager.GetNextIP(useIPv6)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
defer ipManager.ReleaseIP(sourceAddress)
|
||||
defer ip_manager.ReleaseIP(sourceAddress)
|
||||
if useIPv6 {
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
@@ -282,7 +282,7 @@ runcmd:
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
ip_manager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
@@ -477,7 +477,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
}
|
||||
tags = v.youtubeInfo.Snippet.Tags
|
||||
}
|
||||
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package tagsManager
|
||||
package tags_manager
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
@@ -1,4 +1,4 @@
|
||||
package tagsManager
|
||||
package tags_manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
Reference in new issue
Block a user