Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1b2117df5 | ||
|
|
c4207338c8 | ||
|
|
5a01983203 | ||
|
|
ee8eb83d07 | ||
|
|
8d0f762067 | ||
|
|
8fa1482d18 | ||
|
|
00ae404642 | ||
|
|
230bfe4a41 | ||
|
|
df33fb9263 | ||
|
|
d72be1d920 | ||
|
|
7d12a90139 | ||
|
|
e1689a2a6c | ||
|
|
a8a6347d52 | ||
|
|
bdee1b4092 |
No files matched your search
+1
-1
@@ -2,7 +2,7 @@ os: linux
|
||||
dist: bionic
|
||||
language: go
|
||||
go:
|
||||
- 1.16.3
|
||||
- 1.17.x
|
||||
|
||||
install: true
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetVideoInformation(t *testing.T) {
|
||||
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
|
||||
video, err := GetVideoInformation("zj7pXM9gE5M", nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ services:
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:master
|
||||
image: odyseeteam/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
@@ -128,7 +128,7 @@ services:
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
image: odyseeteam/chainquery:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
go 1.17
|
||||
|
||||
module github.com/lbryio/ytsync/v5
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
@@ -6,39 +8,144 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
|
||||
//replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
|
||||
|
||||
require (
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/abadojack/whatlanggo v1.0.1
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
|
||||
github.com/aws/aws-sdk-go v1.27.0
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
|
||||
github.com/aws/aws-sdk-go v1.44.6
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||
github.com/docker/docker v1.13.1
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/lbryio/lbry.go v1.1.2 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20211214213601-4d8e7739d704
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
|
||||
github.com/prometheus/client_golang v1.10.0
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/spf13/afero v1.4.1 // indirect
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/spf13/viper v1.7.1 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/docker/docker v20.10.17+incompatible
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20220805175105-9130630afe5e
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20220730181028-f5d30b1a6e79
|
||||
github.com/mitchellh/go-ps v1.0.0
|
||||
github.com/prometheus/client_golang v1.12.1
|
||||
github.com/shopspring/decimal v1.3.1
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/spf13/cobra v1.4.0
|
||||
github.com/stretchr/testify v1.7.1
|
||||
github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
|
||||
github.com/vbauerster/mpb/v7 v7.0.2
|
||||
gopkg.in/ini.v1 v1.60.2 // indirect
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
|
||||
github.com/vbauerster/mpb/v7 v7.4.1
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.3
|
||||
gotest.tools v2.2.0+incompatible
|
||||
)
|
||||
|
||||
go 1.13
|
||||
require (
|
||||
github.com/Microsoft/go-winio v0.5.1 // indirect
|
||||
github.com/VividCortex/ewma v1.2.0 // indirect
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bluele/gcache v0.0.2 // indirect
|
||||
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 // indirect
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cheekybits/genny v1.0.0 // indirect
|
||||
github.com/docker/distribution v2.8.1+incompatible // indirect
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db // indirect
|
||||
github.com/fatih/structs v1.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/gin-gonic/gin v1.7.7 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-ini/ini v1.48.0 // indirect
|
||||
github.com/go-playground/locales v0.13.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.17.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.4.1 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
github.com/gofrs/uuid v3.2.0+incompatible // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/btree v1.0.1 // indirect
|
||||
github.com/google/go-cmp v0.5.7 // indirect
|
||||
github.com/gorilla/mux v1.8.0 // indirect
|
||||
github.com/gorilla/rpc v1.2.0 // indirect
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/hashicorp/memberlist v0.3.0 // indirect
|
||||
github.com/hashicorp/serf v0.9.7 // indirect
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
|
||||
github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/karrick/godirwalk v1.17.0 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lbryio/chainquery v1.9.0 // indirect
|
||||
github.com/lbryio/lbry.go v1.1.2 // indirect
|
||||
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 // indirect
|
||||
github.com/leodido/go-urn v1.2.0 // indirect
|
||||
github.com/lucas-clemente/quic-go v0.28.1 // indirect
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/marten-seemann/qpack v0.2.1 // indirect
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
|
||||
github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
|
||||
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
|
||||
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.12 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/miekg/dns v1.1.41 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/nxadm/tail v1.4.8 // indirect
|
||||
github.com/onsi/ginkgo v1.16.4 // indirect
|
||||
github.com/onsi/gomega v1.17.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
|
||||
github.com/pelletier/go-toml v1.9.3 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
|
||||
github.com/slack-go/slack v0.10.3 // indirect
|
||||
github.com/spf13/afero v1.4.1 // indirect
|
||||
github.com/spf13/cast v1.4.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.0.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/spf13/viper v1.7.1 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/ugorji/go/codec v1.1.7 // indirect
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
|
||||
github.com/volatiletech/null v8.0.0+incompatible // indirect
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
|
||||
golang.org/x/mod v0.4.2 // indirect
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
|
||||
golang.org/x/tools v0.1.5 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/ini.v1 v1.60.2 // indirect
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
gotest.tools/v3 v3.2.0 // indirect
|
||||
)
|
||||
@@ -32,6 +32,10 @@ var (
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
customFormatter := new(log.TextFormatter)
|
||||
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
|
||||
customFormatter.FullTimestamp = true
|
||||
log.SetFormatter(customFormatter)
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@ func (s *SyncManager) Start() error {
|
||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||
"NotEnoughFunds",
|
||||
"no space left on device",
|
||||
"failure uploading wallet",
|
||||
"there was a problem uploading the wallet",
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
|
||||
+16
-6
@@ -4,6 +4,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/configs"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
@@ -215,13 +216,22 @@ func (s *Sync) uploadWallet() error {
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
start := time.Now()
|
||||
|
||||
for time.Since(start) < 30*time.Minute {
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
time.Sleep(30 * time.Second)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
|
||||
}
|
||||
log.Println("wallet uploaded to S3")
|
||||
|
||||
|
||||
+11
-7
@@ -103,6 +103,9 @@ func (s *Sync) walletSetup() error {
|
||||
videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
if unallocatedVideos < 0 {
|
||||
unallocatedVideos = 0
|
||||
}
|
||||
channelFee := channelClaimAmount
|
||||
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
|
||||
if channelAlreadyClaimed {
|
||||
@@ -131,7 +134,7 @@ func (s *Sync) walletSetup() error {
|
||||
extraLBC := balance - requiredBalance
|
||||
if extraLBC > 5 {
|
||||
sendBackAmount := extraLBC - 1
|
||||
logUtils.SendInfoToSlack("channel %s has %.1f credits which is %.1f more than it should. We should send at least %.1f that back.", s.DbChannelData.ChannelId, balance, extraLBC, sendBackAmount)
|
||||
logUtils.SendInfoToSlack("channel %s has %.1f credits which is %.1f more than it requires (%.1f). We should send at least %.1f that back.", s.DbChannelData.ChannelId, balance, extraLBC, requiredBalance, sendBackAmount)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -372,14 +375,12 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
|
||||
channelUsesOldMetadata := false
|
||||
if channelToUse != nil {
|
||||
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
|
||||
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.Languages) == 0 && s.DbChannelData.Language == "")
|
||||
if !channelUsesOldMetadata {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
channelBidAmount := channelClaimAmount
|
||||
|
||||
balanceResp, err := s.daemon.AccountBalance(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -391,8 +392,8 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
|
||||
err = s.addCredits(channelBidAmount + 0.3)
|
||||
if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
|
||||
err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -430,6 +431,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
var languages []string = nil
|
||||
if s.DbChannelData.Language != "" {
|
||||
languages = []string{s.DbChannelData.Language}
|
||||
}
|
||||
//we don't have this data without the API
|
||||
//if channelInfo.DefaultLanguage != "" {
|
||||
// if channelInfo.DefaultLanguage == "iw" {
|
||||
@@ -466,7 +470,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelClaimAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
})
|
||||
|
||||
+1
-1
@@ -293,7 +293,7 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
|
||||
if updateError != nil {
|
||||
ui.videoStatus.FailureReason = updateError.Error()
|
||||
ui.videoStatus.Status = shared.VideoStatusTranferFailed
|
||||
ui.videoStatus.Status = shared.VideoStatusTransferFailed
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
|
||||
+10
-10
@@ -336,7 +336,7 @@ func (s *Sync) waitForDaemonStart() error {
|
||||
}
|
||||
|
||||
func (s *Sync) stopAndUploadWallet(e *error) {
|
||||
log.Printf("Stopping daemon")
|
||||
log.Println("Stopping daemon")
|
||||
shutdownErr := logUtils.StopDaemon()
|
||||
if shutdownErr != nil {
|
||||
logShutdownError(shutdownErr)
|
||||
@@ -349,16 +349,11 @@ func (s *Sync) stopAndUploadWallet(e *error) {
|
||||
logShutdownError(processDeathError)
|
||||
} else {
|
||||
err := s.uploadWallet()
|
||||
if err != nil {
|
||||
time.Sleep(10 * time.Second)
|
||||
logUtils.SendErrorToSlack("there was a problem uploading the wallet to S3, waiting 10 seconds and retrying: %s", err.Error())
|
||||
err = s.uploadWallet()
|
||||
}
|
||||
if err != nil {
|
||||
if *e == nil {
|
||||
*e = err
|
||||
} else {
|
||||
*e = errors.Prefix(fmt.Sprintf("failure uploading wallet: %s + original error", errors.FullTrace(err)), *e)
|
||||
*e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
|
||||
}
|
||||
}
|
||||
err = s.uploadBlockchainDB()
|
||||
@@ -501,7 +496,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
||||
_, isOwnClaim := ownClaimsInfo[videoID]
|
||||
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
|
||||
transferStatusMismatch := sv.Transferred != transferred
|
||||
transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
|
||||
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
||||
@@ -562,7 +557,11 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
if sv.Transferred || sv.IsLbryFirst {
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
|
||||
searchResponse, err := s.daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
|
||||
ClaimID: &sv.ClaimID,
|
||||
Page: 1,
|
||||
PageSize: 20,
|
||||
})
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
continue
|
||||
@@ -678,7 +677,8 @@ func (s *Sync) checkIntegrity() error {
|
||||
|
||||
if pubsOnWallet > pubsOnDB { //This case should never happen
|
||||
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
|
||||
return errors.Err("not all published videos are in the database")
|
||||
//we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
|
||||
//return errors.Err("not all published videos are in the database")
|
||||
}
|
||||
if pubsOnWallet < pubsOnDB {
|
||||
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
|
||||
|
||||
+24
-48
@@ -61,12 +61,9 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
|
||||
"channel_id": {cliFlags.ChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -124,12 +121,9 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
"auth_token": {a.ApiToken},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -172,12 +166,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
}
|
||||
res, err := http.PostForm(endpoint, params)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
return nil, nil, errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -222,12 +213,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
"channel_claim_id": {channelClaimID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -266,12 +254,9 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
return errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -332,12 +317,9 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
return errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -374,12 +356,9 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
return "", errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
@@ -428,12 +407,9 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
|
||||
+13
-9
@@ -26,6 +26,7 @@ type YoutubeChannel struct {
|
||||
SizeLimit int `json:"size_limit"`
|
||||
LastUploadedVideo string `json:"last_uploaded_video"`
|
||||
WipeDB bool `json:"wipe_db"`
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
type PublishAddress struct {
|
||||
@@ -153,7 +154,7 @@ func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
|
||||
800: 250,
|
||||
600: 200,
|
||||
200: 80,
|
||||
100: 50,
|
||||
100: 20,
|
||||
1: 10,
|
||||
}
|
||||
videosToSync := 0
|
||||
@@ -190,22 +191,25 @@ const (
|
||||
StatusSynced = "synced" // done
|
||||
StatusWipeDb = "pendingdbwipe" // in sync queue. lbryum database will be pruned
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
StatusAgeRestricted = "agerestricted" // one or more videos are age restricted and should be reprocessed with special keys
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb, StatusAgeRestricted}
|
||||
|
||||
const LatestMetadataVersion = 2
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
VideoStatusTranferFailed = "transferfailed"
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
VideoStatusTransferFailed = "transferfailed"
|
||||
)
|
||||
|
||||
var VideoSyncStatuses = []string{VideoStatusPublished, VideoStatusFailed, VideoStatusUpgradeFailed, VideoStatusUnpublished, VideoStatusTransferFailed}
|
||||
|
||||
const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
|
||||
+23
-13
@@ -16,17 +16,12 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/abadojack/whatlanggo"
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
"github.com/vbauerster/mpb/v7/decor"
|
||||
"gopkg.in/vansante/go-ffprobe.v2"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
@@ -37,8 +32,12 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/abadojack/whatlanggo"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
"github.com/vbauerster/mpb/v7/decor"
|
||||
"gopkg.in/vansante/go-ffprobe.v2"
|
||||
)
|
||||
|
||||
type YoutubeVideo struct {
|
||||
@@ -176,7 +175,7 @@ func (v *YoutubeVideo) getFullPath() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
maxLength := 2800
|
||||
maxLength := 6500
|
||||
description := strings.TrimSpace(v.description)
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
@@ -321,7 +320,7 @@ func (v *YoutubeVideo) download() error {
|
||||
//speedThrottleRetries := 3
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=380][format_id!=251][format_id!=256][format_id!=327][format_id!=328]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
//if speedThrottleRetries > 0 {
|
||||
// speedThrottleRetries--
|
||||
@@ -515,21 +514,28 @@ func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.T
|
||||
bar.Completed()
|
||||
bar.Abort(true)
|
||||
}()
|
||||
origSize := int64(0)
|
||||
lastUpdate := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-done.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
var err error
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
bar.SetCurrent(size)
|
||||
if size > int64(videoSize+audioSize) {
|
||||
bar.SetTotal(size+2048, false)
|
||||
if size > origSize {
|
||||
origSize = size
|
||||
bar.SetCurrent(size)
|
||||
if size > int64(videoSize+audioSize) {
|
||||
bar.SetTotal(size+2048, false)
|
||||
}
|
||||
bar.DecoratorEwmaUpdate(time.Since(lastUpdate))
|
||||
lastUpdate = time.Now()
|
||||
}
|
||||
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -862,7 +868,11 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
|
||||
c, err := daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
|
||||
ClaimID: &existingVideoData.ClaimID,
|
||||
Page: 1,
|
||||
PageSize: 20,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user