Compare commits

...
23 Commits
Author SHA1 Message Date
Niko Storni c1b2117df5 fix vuln 2022-08-09 22:27:00 +02:00
Niko Storni c4207338c8 fix const bug
update dependencies
2022-08-09 22:17:03 +02:00
Niko Storni 5a01983203 add language to channels
improve logging (timestamps)
retry wallet uploads for 30 minutes
don't fail if the db isn't tracking all publishes
2022-08-09 22:11:42 +02:00
Niko Storni ee8eb83d07 fix dependency vuln 2022-05-10 23:36:00 +02:00
Niko Storni 8d0f762067 change description length 2022-05-10 23:09:23 +02:00
Niko Storni 8fa1482d18 adjust limits 2022-05-09 20:23:44 +02:00
Niko Storni 00ae404642 exclude audio ac-3 codec not working in chrome 2022-05-09 20:05:05 +02:00
Niko Storni 230bfe4a41 fix go version in travis 2022-05-04 19:22:19 +02:00
Niko Storni df33fb9263 fix api failures
update dependencies
fix e2e
2022-05-04 19:17:36 +02:00
Niko Storni d72be1d920 add new status 2022-03-03 18:59:32 +01:00
Niko Storni 7d12a90139 avoid ec-3 audio codec 2022-02-09 17:06:51 +01:00
Niko e1689a2a6c Merge pull request #116 from e4drcf/synced-video-statuses
add shared video sync statuses array for validation purposes
2022-02-07 20:03:29 +01:00
Ivan a8a6347d52 add shared video sync statuses array for validation purposes 2022-02-07 20:17:41 +02:00
Niko Storni bdee1b4092 fix math to avoid negative balances 2022-01-26 07:43:09 +01:00
Niko Storni 0d0d39380c fix spend amounts to save credits 2022-01-26 07:11:26 +01:00
Niko Storni 7ff1a009da remove unused params 2022-01-14 18:49:02 +01:00
Niko Storni e3a332c7e1 upgrade dependencies 2022-01-14 18:10:58 +01:00
Niko Storni 33ee6e4b94 Merge remote-tracking branch 'origin/metadata_fix' 2022-01-14 17:20:22 +01:00
Niko Storni f6cde976a6 fix metadata? 2022-01-06 15:15:04 +01:00
Niko Storni 17944fa46a refactor get video time
remove broken time lookup
refactor quite some code
2021-12-30 13:17:11 -05:00
Niko Storni 3c18ae8de2 add checks for buggy livestreams 2021-12-29 17:47:46 -05:00
Niko Storni 84790720ff improve error handling
retry wallet uploads on failure
2021-12-02 16:59:14 +01:00
Niko 23690731af fix bug when uploading wallet 2021-11-30 02:53:59 +01:00
20 changed files with 939 additions and 481 deletions

No files matched your search

+1 -1
View File
@@ -2,7 +2,7 @@ os: linux
dist: bionic
language: go
go:
- 1.16.3
- 1.17.x
install: true
+23
View File
@@ -1,10 +1,14 @@
package configs
import (
"os"
"regexp"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
"github.com/tkanos/gonfig"
)
@@ -50,3 +54,22 @@ func (s *S3Configs) GetS3AWSConfig() *aws.Config {
S3ForcePathStyle: aws.Bool(true),
}
}
func (c *Configs) GetHostname() string {
var hostname string
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync_unknown"
}
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err == nil {
hostname = reg.ReplaceAllString(hostname, "_")
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
return hostname
}
+2 -84
View File
@@ -50,7 +50,7 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
args := []string{
"--skip-download",
"--write-info-json",
@@ -80,50 +80,6 @@ func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Ch
return nil, errors.Err(err)
}
// now get an accurate time
const maxTries = 5
tries := 0
GetTime:
tries++
t, err := getUploadTime(config, videoID, ip, video.UploadDate)
if err != nil {
//slack(":warning: Upload time error: %v", err)
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
err := triggerScrape(videoID, ip)
if err == nil {
time.Sleep(2 * time.Second) // let them scrape it
goto GetTime
} else {
//slack("triggering scrape returned error: %v", err)
}
} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
if t == "" {
return nil, errors.Err(err)
} else {
t = "" //TODO: get rid of the other piece below?
}
}
// do fallback below
}
//slack("After all that, upload time for %s is %s", videoID, t)
if t != "" {
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
if err != nil {
return nil, errors.Err(err)
}
//slack(":exclamation: Got an accurate time for %s", videoID)
video.UploadDateForReal = parsed
} else { //TODO: this is the piece that isn't needed!
slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
// fall back to UploadDate from youtube-dl
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
if err != nil {
return nil, err
}
}
return video, nil
}
@@ -213,45 +169,7 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
}
}
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
return ytdlUploadDate.Format(releaseTimeFormat), nil
}
client := getClient(ip)
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
req.Header.Set("User-Agent", ChromeUA)
res, err := client.Do(req)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
defer res.Body.Close()
var uploadTime struct {
Time string `json:"video_upload_time"`
Message string `json:"message"`
Status string `json:"status"`
}
err = json.NewDecoder(res.Body).Decode(&uploadTime)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
if uploadTime.Status == "ERROR1" {
return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
}
if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
}
if uploadTime.Time == "" {
return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
}
return uploadTime.Time, nil
return ytdlUploadDate.Format(releaseTimeFormat), nil
}
func getClient(ip *net.TCPAddr) *http.Client {
+1 -1
View File
@@ -19,7 +19,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
}
func TestGetVideoInformation(t *testing.T) {
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
video, err := GetVideoInformation("zj7pXM9gE5M", nil, nil)
if err != nil {
logrus.Error(err)
}
+142 -92
View File
@@ -1,105 +1,120 @@
package ytdl
import (
"math"
"time"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/sirupsen/logrus"
)
type YtdlVideo struct {
UploadDate string `json:"upload_date"`
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
Extractor string `json:"extractor"`
Series interface{} `json:"series"`
Format string `json:"format"`
Vbr interface{} `json:"vbr"`
Chapters interface{} `json:"chapters"`
Height int `json:"height"`
LikeCount interface{} `json:"like_count"`
Duration int `json:"duration"`
Fulltitle string `json:"fulltitle"`
PlaylistIndex interface{} `json:"playlist_index"`
Album interface{} `json:"album"`
ViewCount int `json:"view_count"`
Playlist interface{} `json:"playlist"`
Title string `json:"title"`
Filename string `json:"_filename"`
Creator interface{} `json:"creator"`
Ext string `json:"ext"`
ID string `json:"id"`
DislikeCount interface{} `json:"dislike_count"`
AverageRating float64 `json:"average_rating"`
Abr float64 `json:"abr"`
UploaderURL string `json:"uploader_url"`
Categories []string `json:"categories"`
Fps float64 `json:"fps"`
StretchedRatio interface{} `json:"stretched_ratio"`
SeasonNumber interface{} `json:"season_number"`
Annotations interface{} `json:"annotations"`
WebpageURLBasename string `json:"webpage_url_basename"`
Acodec string `json:"acodec"`
DisplayID string `json:"display_id"`
//RequestedFormats []RequestedFormat `json:"requested_formats"`
//AutomaticCaptions struct{} `json:"automatic_captions"`
ID string `json:"id"`
Title string `json:"title"`
Formats []struct {
FormatId string `json:"format_id"`
FormatNote string `json:"format_note"`
Ext string `json:"ext"`
Protocol string `json:"protocol"`
Acodec string `json:"acodec"`
Vcodec string `json:"vcodec"`
Url string `json:"url"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Fragments []struct {
Path string `json:"path"`
Duration float64 `json:"duration"`
} `json:"fragments,omitempty"`
AudioExt string `json:"audio_ext"`
VideoExt string `json:"video_ext"`
Format string `json:"format"`
Resolution string `json:"resolution"`
HttpHeaders struct {
UserAgent string `json:"User-Agent"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Asr float64 `json:"asr,omitempty"`
Filesize int64 `json:"filesize,omitempty"`
SourcePreference int `json:"source_preference,omitempty"`
Quality int `json:"quality,omitempty"`
Tbr float64 `json:"tbr,omitempty"`
Language string `json:"language,omitempty"`
LanguagePreference int `json:"language_preference,omitempty"`
Abr float64 `json:"abr,omitempty"`
DownloaderOptions struct {
HttpChunkSize int `json:"http_chunk_size"`
} `json:"downloader_options,omitempty"`
Container string `json:"container,omitempty"`
Fps float64 `json:"fps,omitempty"`
DynamicRange string `json:"dynamic_range,omitempty"`
Vbr float64 `json:"vbr,omitempty"`
FilesizeApprox float64 `json:"filesize_approx,omitempty"`
} `json:"formats"`
Thumbnails []Thumbnail `json:"thumbnails"`
Thumbnail string `json:"thumbnail"`
Description string `json:"description"`
Tags []string `json:"tags"`
Track interface{} `json:"track"`
RequestedSubtitles interface{} `json:"requested_subtitles"`
StartTime interface{} `json:"start_time"`
UploadDate string `json:"upload_date"`
uploadDateForReal *time.Time `json:"upload_date_for_real"`
Uploader string `json:"uploader"`
UploaderId string `json:"uploader_id"`
UploaderUrl string `json:"uploader_url"`
ChannelID string `json:"channel_id"`
ChannelUrl string `json:"channel_url"`
Duration int `json:"duration"`
ViewCount int `json:"view_count"`
AgeLimit int `json:"age_limit"`
WebpageUrl string `json:"webpage_url"`
Categories []string `json:"categories"`
Tags []string `json:"tags"`
PlayableInEmbed bool `json:"playable_in_embed"`
IsLive bool `json:"is_live"`
WasLive bool `json:"was_live"`
LiveStatus string `json:"live_status"`
ReleaseTimestamp int64 `json:"release_timestamp"`
LikeCount int `json:"like_count"`
Channel string `json:"channel"`
Availability string `json:"availability"`
WebpageUrlBasename string `json:"webpage_url_basename"`
WebpageUrlDomain string `json:"webpage_url_domain"`
Extractor string `json:"extractor"`
ExtractorKey string `json:"extractor_key"`
FormatID string `json:"format_id"`
EpisodeNumber interface{} `json:"episode_number"`
UploaderID string `json:"uploader_id"`
//Subtitles struct{} `json:"subtitles"`
ReleaseYear interface{} `json:"release_year"`
Thumbnails []Thumbnail `json:"thumbnails"`
License interface{} `json:"license"`
Artist interface{} `json:"artist"`
AgeLimit int `json:"age_limit"`
ReleaseDate interface{} `json:"release_date"`
AltTitle interface{} `json:"alt_title"`
Thumbnail string `json:"thumbnail"`
ChannelID string `json:"channel_id"`
IsLive interface{} `json:"is_live"`
Width int `json:"width"`
EndTime interface{} `json:"end_time"`
WebpageURL string `json:"webpage_url"`
Formats []Format `json:"formats"`
ChannelURL string `json:"channel_url"`
Resolution interface{} `json:"resolution"`
Vcodec string `json:"vcodec"`
}
type RequestedFormat struct {
Asr interface{} `json:"asr"`
Tbr float64 `json:"tbr"`
Container string `json:"container"`
Language interface{} `json:"language"`
Format string `json:"format"`
URL string `json:"url"`
Vcodec string `json:"vcodec"`
FormatNote string `json:"format_note"`
Height int `json:"height"`
Width int `json:"width"`
Ext string `json:"ext"`
FragmentBaseURL string `json:"fragment_base_url"`
Filesize interface{} `json:"filesize"`
Fps float64 `json:"fps"`
ManifestURL string `json:"manifest_url"`
Protocol string `json:"protocol"`
FormatID string `json:"format_id"`
HTTPHeaders struct {
AcceptCharset string `json:"Accept-Charset"`
AcceptLanguage string `json:"Accept-Language"`
AcceptEncoding string `json:"Accept-Encoding"`
Accept string `json:"Accept"`
DisplayId string `json:"display_id"`
DurationString string `json:"duration_string"`
ReleaseDate string `json:"release_date"`
Asr float64 `json:"asr"`
FormatId string `json:"format_id"`
FormatNote string `json:"format_note"`
SourcePreference int `json:"source_preference"`
Fps float64 `json:"fps"`
Height int `json:"height"`
Quality int `json:"quality"`
Tbr float64 `json:"tbr"`
Url string `json:"url"`
Width int `json:"width"`
Language string `json:"language"`
LanguagePreference int `json:"language_preference"`
Ext string `json:"ext"`
Vcodec string `json:"vcodec"`
Acodec string `json:"acodec"`
DynamicRange string `json:"dynamic_range"`
Protocol string `json:"protocol"`
VideoExt string `json:"video_ext"`
AudioExt string `json:"audio_ext"`
Vbr float64 `json:"vbr"`
Abr float64 `json:"abr"`
Format string `json:"format"`
Resolution string `json:"resolution"`
FilesizeApprox float64 `json:"filesize_approx"`
HttpHeaders struct {
UserAgent string `json:"User-Agent"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Fragments []struct {
Path string `json:"path"`
Duration float64 `json:"duration,omitempty"`
} `json:"fragments"`
Acodec string `json:"acodec"`
Abr int `json:"abr,omitempty"`
Fulltitle string `json:"fulltitle"`
Epoch int `json:"epoch"`
}
type Format struct {
@@ -135,10 +150,11 @@ type Format struct {
type Thumbnail struct {
URL string `json:"url"`
Width int `json:"width"`
Resolution string `json:"resolution"`
Preference int `json:"preference"`
ID string `json:"id"`
Height int `json:"height"`
Width int `json:"width"`
Resolution string `json:"resolution"`
}
type HTTPHeaders struct {
@@ -148,3 +164,37 @@ type HTTPHeaders struct {
Accept string `json:"Accept"`
UserAgent string `json:"User-Agent"`
}
func (v *YtdlVideo) GetUploadTime() time.Time {
if v.uploadDateForReal != nil {
return *v.uploadDateForReal
}
release, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
if err != nil {
logrus.Error(err)
}
ytdlUploadDate, err := time.Parse("20060102", v.UploadDate)
if err != nil {
logrus.Error(err)
}
if v.ReleaseTimestamp != 0 {
ytdlUploadDate = time.Unix(v.ReleaseTimestamp, 0)
}
if release != nil {
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
if err == nil {
hoursDiff := math.Abs(sqlTime.Sub(ytdlUploadDate).Hours())
if hoursDiff > 48 {
logrus.Infof("upload day from APIs differs from the ytdl one by more than 2 days.")
} else {
v.uploadDateForReal = &sqlTime
return sqlTime
}
} else {
logrus.Error(err)
}
}
v.uploadDateForReal = &ytdlUploadDate
return ytdlUploadDate
}
+4 -3
View File
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:latest-release
image: lbry/wallet-server:v0.101.1
restart: always
environment:
- DB_DIRECTORY=/database
@@ -81,6 +81,7 @@ services:
- walletserver
environment:
- LBRY_STREAMING_SERVER=0.0.0.0:5280
- LBRY_FEE_PER_NAME_CHAR=0
volumes:
- "./persist/.lbrynet:/home/lbrynet"
- ".:/etc/lbry" #Put your daemon_settings.yml here
@@ -109,7 +110,7 @@ services:
## Internal APIs ##
###################
internalapis:
image: lbry/internal-apis:master
image: odyseeteam/internal-apis:master
restart: "no"
ports:
- "15400:8080"
@@ -127,7 +128,7 @@ services:
## Chainquery ##
################
chainquery:
image: lbry/chainquery:master
image: odyseeteam/chainquery:master
restart: "no"
ports:
- 6300:6300
+137 -29
View File
@@ -1,3 +1,5 @@
go 1.17
module github.com/lbryio/ytsync/v5
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
@@ -6,38 +8,144 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
//replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
require (
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/abadojack/whatlanggo v1.0.1
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
github.com/aws/aws-sdk-go v1.25.9
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
github.com/aws/aws-sdk-go v1.44.6
github.com/davecgh/go-spew v1.1.1
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
github.com/prometheus/client_golang v0.9.3
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0
github.com/docker/docker v20.10.17+incompatible
github.com/lbryio/lbry.go/v2 v2.7.2-0.20220805175105-9130630afe5e
github.com/lbryio/reflector.go v1.1.3-0.20220730181028-f5d30b1a6e79
github.com/mitchellh/go-ps v1.0.0
github.com/prometheus/client_golang v1.12.1
github.com/shopspring/decimal v1.3.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.1
github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
github.com/vbauerster/mpb/v7 v7.0.2
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/ini.v1 v1.60.2 // indirect
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
github.com/vbauerster/mpb/v7 v7.4.1
gopkg.in/vansante/go-ffprobe.v2 v2.0.3
gotest.tools v2.2.0+incompatible
)
go 1.13
require (
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 // indirect
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.7.7 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-ini/ini v1.48.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.3.0 // indirect
github.com/hashicorp/serf v0.9.7 // indirect
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/karrick/godirwalk v1.17.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lbryio/chainquery v1.9.0 // indirect
github.com/lbryio/lbry.go v1.1.2 // indirect
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/lucas-clemente/quic-go v0.28.1 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/marten-seemann/qpack v0.2.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.17.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/slack-go/slack v0.10.3 // indirect
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.7.1 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
github.com/volatiletech/null v8.0.0+incompatible // indirect
github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.1.5 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.60.2 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gotest.tools/v3 v3.2.0 // indirect
)
+477 -125
View File
File diff suppressed because it is too large. Load diff
+5 -21
View File
@@ -9,7 +9,6 @@ import (
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
@@ -33,6 +32,10 @@ var (
func main() {
rand.Seed(time.Now().UnixNano())
log.SetLevel(log.DebugLevel)
customFormatter := new(log.TextFormatter)
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
customFormatter.FullTimestamp = true
log.SetFormatter(customFormatter)
http.Handle("/metrics", promhttp.Handler())
go func() {
log.Error(http.ListenAndServe(":2112", nil))
@@ -75,22 +78,11 @@ func ytSync(cmd *cobra.Command, args []string) {
if err != nil {
log.Fatalf("could not parse configuration file: %s", errors.FullTrace(err))
}
var hostname string
if configs.Configuration.SlackToken == "" {
log.Error("A slack token was not present in the config! Slack messages disabled!")
} else {
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync-unknown"
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, hostname)
util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, configs.Configuration.GetHostname())
}
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
@@ -131,17 +123,9 @@ func ytSync(cmd *cobra.Command, args []string) {
blobsDir := ytUtils.GetBlobsDir()
apiConfig := &sdk.APIConfig{
ApiURL: configs.Configuration.InternalApisEndpoint,
ApiToken: configs.Configuration.InternalApisAuthToken,
HostName: hostname,
}
sm := manager.NewSyncManager(
cliFlags,
blobsDir,
configs.Configuration.LbrycrdString,
apiConfig,
)
err = sm.Start()
if err != nil {
+5 -4
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/lbryio/ytsync/v5/blobs_reflector"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
@@ -29,12 +30,12 @@ type SyncManager struct {
channelsToSync []Sync
}
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, apiConfig *sdk.APIConfig) *SyncManager {
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir string) *SyncManager {
return &SyncManager{
CliFlags: cliFlags,
blobsDir: blobsDir,
LbrycrdDsn: lbrycrdDsn,
ApiConfig: apiConfig,
LbrycrdDsn: configs.Configuration.LbrycrdString,
ApiConfig: sdk.GetAPIsConfigs(),
}
}
func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
@@ -137,7 +138,7 @@ func (s *SyncManager) Start() error {
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
"NotEnoughFunds",
"no space left on device",
"failure uploading wallet",
"there was a problem uploading the wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
+16 -6
View File
@@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/util"
@@ -215,13 +216,22 @@ func (s *Sync) uploadWallet() error {
}
defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Key: key,
Body: file,
})
start := time.Now()
for time.Since(start) < 30*time.Minute {
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Key: key,
Body: file,
})
if err != nil {
time.Sleep(30 * time.Second)
continue
}
break
}
if err != nil {
return err
return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
}
log.Println("wallet uploaded to S3")
+17 -7
View File
@@ -103,6 +103,9 @@ func (s *Sync) walletSetup() error {
videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
if unallocatedVideos < 0 {
unallocatedVideos = 0
}
channelFee := channelClaimAmount
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
if channelAlreadyClaimed {
@@ -110,7 +113,7 @@ func (s *Sync) walletSetup() error {
}
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
if s.Manager.CliFlags.UpgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
requiredBalance += float64(notUpgradedCount) * estimatedMaxTxFee
}
refillAmount := 0.0
@@ -127,6 +130,12 @@ func (s *Sync) walletSetup() error {
if err != nil {
return errors.Err(err)
}
} else if balance > requiredBalance {
extraLBC := balance - requiredBalance
if extraLBC > 5 {
sendBackAmount := extraLBC - 1
logUtils.SendInfoToSlack("channel %s has %.1f credits which is %.1f more than it requires (%.1f). We should send at least %.1f that back.", s.DbChannelData.ChannelId, balance, extraLBC, requiredBalance, sendBackAmount)
}
}
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
@@ -366,14 +375,12 @@ func (s *Sync) ensureChannelOwnership() error {
channelUsesOldMetadata := false
if channelToUse != nil {
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.Languages) == 0 && s.DbChannelData.Language == "")
if !channelUsesOldMetadata {
return nil
}
}
channelBidAmount := channelClaimAmount
balanceResp, err := s.daemon.AccountBalance(nil)
if err != nil {
return err
@@ -385,8 +392,8 @@ func (s *Sync) ensureChannelOwnership() error {
return errors.Err(err)
}
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
err = s.addCredits(channelBidAmount + 0.3)
if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
if err != nil {
return err
}
@@ -424,6 +431,9 @@ func (s *Sync) ensureChannelOwnership() error {
}
var languages []string = nil
if s.DbChannelData.Language != "" {
languages = []string{s.DbChannelData.Language}
}
//we don't have this data without the API
//if channelInfo.DefaultLanguage != "" {
// if channelInfo.DefaultLanguage == "iw" {
@@ -460,7 +470,7 @@ func (s *Sync) ensureChannelOwnership() error {
return nil
}
} else {
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelClaimAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
})
+2 -2
View File
@@ -241,7 +241,7 @@ func transferVideos(s *Sync) error {
},
},
},
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
Bid: util.PtrToString(fmt.Sprintf("%.5f", publishAmount/2.)),
}
videoStatus := shared.VideoStatus{
ChannelID: s.DbChannelData.ChannelId,
@@ -293,7 +293,7 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
if updateError != nil {
ui.videoStatus.FailureReason = updateError.Error()
ui.videoStatus.Status = shared.VideoStatusTranferFailed
ui.videoStatus.Status = shared.VideoStatusTransferFailed
ui.videoStatus.IsTransferred = util.PtrToBool(false)
} else {
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
+16 -10
View File
@@ -33,11 +33,10 @@ import (
const (
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.1
estimatedMaxTxFee = 0.0015
minimumAccountBalance = 1.0
minimumRefillAmount = 1
publishAmount = 0.01
maxReasonLength = 500
publishAmount = 0.002
)
// Sync stores the options that control how syncing happens
@@ -286,6 +285,8 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
"interrupted during daemon startup",
"interrupted by user",
"use --skip-space-check to ignore",
"failure uploading blockchain DB",
"default_wallet already exists",
}
dbWipeConditions := []string{
"Missing inputs",
@@ -335,7 +336,7 @@ func (s *Sync) waitForDaemonStart() error {
}
func (s *Sync) stopAndUploadWallet(e *error) {
log.Printf("Stopping daemon")
log.Println("Stopping daemon")
shutdownErr := logUtils.StopDaemon()
if shutdownErr != nil {
logShutdownError(shutdownErr)
@@ -350,9 +351,9 @@ func (s *Sync) stopAndUploadWallet(e *error) {
err := s.uploadWallet()
if err != nil {
if *e == nil {
e = &err
*e = err
} else {
*e = errors.Prefix("failure uploading wallet", *e)
*e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
}
}
err = s.uploadBlockchainDB()
@@ -495,7 +496,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
claimMarkedUnpublished := claimInDatabase && !sv.Published
_, isOwnClaim := ownClaimsInfo[videoID]
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
transferStatusMismatch := sv.Transferred != transferred
transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -556,7 +557,11 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
if sv.Transferred || sv.IsLbryFirst {
_, ok := allClaimsInfo[vID]
if !ok && sv.Published {
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
searchResponse, err := s.daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
ClaimID: &sv.ClaimID,
Page: 1,
PageSize: 20,
})
if err != nil {
log.Error(err.Error())
continue
@@ -672,7 +677,8 @@ func (s *Sync) checkIntegrity() error {
if pubsOnWallet > pubsOnDB { //This case should never happen
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
return errors.Err("not all published videos are in the database")
//we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
//return errors.Err("not all published videos are in the database")
}
if pubsOnWallet < pubsOnDB {
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
@@ -861,7 +867,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
return err
}
videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
videos, err := ytapi.GetVideosToSync(s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
VideoDir: s.videoDirectory,
Stopper: s.grp,
IPPool: ipPool,
+2 -16
View File
@@ -1,31 +1,17 @@
package metrics
import (
"os"
"regexp"
"github.com/lbryio/ytsync/v5/configs"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
)
var (
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "ytsync",
Subsystem: getHostname(),
Subsystem: configs.Configuration.GetHostname(),
Name: "duration",
Help: "The durations of the individual modules",
}, []string{"path"})
)
func getHostname() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "ytsync_unknown"
}
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err != nil {
log.Fatal(err)
}
return reg.ReplaceAllString(hostname, "_")
}
+38 -48
View File
@@ -13,6 +13,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/null"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/util"
@@ -30,6 +31,19 @@ type APIConfig struct {
HostName string
}
var instance *APIConfig
func GetAPIsConfigs() *APIConfig {
if instance == nil {
instance = &APIConfig{
ApiURL: configs.Configuration.InternalApisEndpoint,
ApiToken: configs.Configuration.InternalApisAuthToken,
HostName: configs.Configuration.GetHostname(),
}
}
return instance
}
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
type apiJobsResponse struct {
Success bool `json:"success"`
@@ -47,12 +61,9 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
"channel_id": {cliFlags.ChannelID},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cliFlags)
}
return nil, errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cliFlags)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -110,12 +121,9 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
"auth_token": {a.ApiToken},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
return errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -158,12 +166,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
}
res, err := http.PostForm(endpoint, params)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
return nil, nil, errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -208,12 +213,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
"channel_claim_id": {channelClaimID},
})
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
return errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -252,12 +254,9 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
return errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -318,12 +317,9 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
return errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -360,12 +356,9 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.VideoState(videoID)
}
return "", errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.VideoState(videoID)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
@@ -414,12 +407,9 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.GetReleasedDate(videoID)
}
return nil, errors.Err(err)
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
time.Sleep(30 * time.Second)
return a.GetReleasedDate(videoID)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
+14 -9
View File
@@ -26,6 +26,7 @@ type YoutubeChannel struct {
SizeLimit int `json:"size_limit"`
LastUploadedVideo string `json:"last_uploaded_video"`
WipeDB bool `json:"wipe_db"`
Language string `json:"language"`
}
type PublishAddress struct {
@@ -89,6 +90,7 @@ var ErrorsNoRetry = []string{
"Premiere will begin shortly",
"cannot unmarshal number 0.0",
"default youtube thumbnail found",
"livestream is likely bugged",
}
var WalletErrors = []string{
"Not enough funds to cover this transaction",
@@ -152,7 +154,7 @@ func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
800: 250,
600: 200,
200: 80,
100: 50,
100: 20,
1: 10,
}
videosToSync := 0
@@ -189,22 +191,25 @@ const (
StatusSynced = "synced" // done
StatusWipeDb = "pendingdbwipe" // in sync queue. lbryum database will be pruned
StatusFailed = "failed"
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
StatusAgeRestricted = "agerestricted" // one or more videos are age restricted and should be reprocessed with special keys
)
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb, StatusAgeRestricted}
const LatestMetadataVersion = 2
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
VideoStatusTranferFailed = "transferfailed"
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
VideoStatusTransferFailed = "transferfailed"
)
var VideoSyncStatuses = []string{VideoStatusPublished, VideoStatusFailed, VideoStatusUpgradeFailed, VideoStatusUnpublished, VideoStatusTransferFailed}
const (
TransferStateNotTouched = iota
TransferStatePending
+28 -14
View File
@@ -16,17 +16,12 @@ import (
"syscall"
"time"
"github.com/abadojack/whatlanggo"
"github.com/lbryio/ytsync/v5/downloader"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/ytsync/v5/shared"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
"gopkg.in/vansante/go-ffprobe.v2"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/lbryio/ytsync/v5/timing"
@@ -37,8 +32,12 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/abadojack/whatlanggo"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
"gopkg.in/vansante/go-ffprobe.v2"
)
type YoutubeVideo struct {
@@ -107,7 +106,7 @@ func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPositi
title: videoData.Title,
description: videoData.Description,
playlistPosition: playlistPosition,
publishedAt: videoData.UploadDateForReal,
publishedAt: videoData.GetUploadTime(),
dir: directory,
youtubeInfo: videoData,
mocked: false,
@@ -176,7 +175,7 @@ func (v *YoutubeVideo) getFullPath() string {
}
func (v *YoutubeVideo) getAbbrevDescription() string {
maxLength := 2800
maxLength := 6500
description := strings.TrimSpace(v.description)
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
@@ -321,7 +320,7 @@ func (v *YoutubeVideo) download() error {
//speedThrottleRetries := 3
for i := 0; i < len(qualities); i++ {
quality := qualities[i]
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=380][format_id!=251][format_id!=256][format_id!=327][format_id!=328]")
argsWithFilters = append(argsWithFilters, userAgent...)
//if speedThrottleRetries > 0 {
// speedThrottleRetries--
@@ -515,21 +514,28 @@ func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.T
bar.Completed()
bar.Abort(true)
}()
origSize := int64(0)
lastUpdate := time.Now()
for {
select {
case <-done.Ch():
return
case <-ticker.C:
var err error
size, err := logUtils.DirSize(v.videoDir())
if err != nil {
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
return
}
bar.SetCurrent(size)
if size > int64(videoSize+audioSize) {
bar.SetTotal(size+2048, false)
if size > origSize {
origSize = size
bar.SetCurrent(size)
if size > int64(videoSize+audioSize) {
bar.SetTotal(size+2048, false)
}
bar.DecoratorEwmaUpdate(time.Since(lastUpdate))
lastUpdate = time.Now()
}
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
}
}
}()
@@ -789,6 +795,10 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
return nil, errors.Err("video is too short to process")
}
buggedLivestream := v.youtubeInfo.WasLive && time.Unix(v.youtubeInfo.ReleaseTimestamp, 0).After(time.Now().AddDate(0, 0, -1)) && int(dur.Seconds())%7200 == 0
if buggedLivestream {
return nil, errors.Err("livestream is likely bugged as it was published less than 24 hours ago and has a length of %s", dur.String())
}
for {
err = v.download()
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
@@ -858,7 +868,11 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
c, err := daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
ClaimID: &existingVideoData.ClaimID,
Page: 1,
PageSize: 20,
})
if err != nil {
return nil, errors.Err(err)
}
+4
View File
@@ -14,7 +14,9 @@ func SendErrorToSlack(format string, a ...interface{}) {
message = fmt.Sprintf(format, a...)
}
log.Errorln(message)
log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
err := util.SendToSlack(":sos: ```" + message + "```")
log.SetLevel(log.DebugLevel)
if err != nil {
log.Errorln(err)
}
@@ -27,7 +29,9 @@ func SendInfoToSlack(format string, a ...interface{}) {
message = fmt.Sprintf(format, a...)
}
log.Infoln(message)
log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
err := util.SendToSlack(":information_source: " + message)
log.SetLevel(log.DebugLevel)
if err != nil {
log.Errorln(err)
}
+5 -9
View File
@@ -55,7 +55,7 @@ type VideoParams struct {
var mostRecentlyFailedChannel string // TODO: fix this hack!
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
var videos []Video
if quickSync && maxVideos > 50 {
maxVideos = 50
@@ -94,7 +94,7 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
mostRecentlyFailedChannel = channelID
}
vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
vids, err := getVideos(channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
if err != nil {
return nil, err
}
@@ -203,7 +203,8 @@ func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
return &decodedResponse, nil
}
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
config := sdk.GetAPIsConfigs()
var videos []*ytdl.YtdlVideo
for _, videoID := range videoIDs {
if len(videoID) < 5 {
@@ -215,11 +216,6 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
default:
}
//ip, err := ipPool.GetIP(videoID)
//if err != nil {
// return nil, err
//}
//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
state, err := config.VideoState(videoID)
if err != nil {
return nil, errors.Err(err)
@@ -227,7 +223,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
if state == "published" {
continue
}
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
video, err := downloader.GetVideoInformation(videoID, stopChan, ipPool)
if err != nil {
errSDK := config.MarkVideoStatus(shared.VideoStatus{
ChannelID: channelID,