Compare commits

...
5 Commits
Author SHA1 Message Date
Niko Storni 77988c1682 woops 2022-08-10 22:01:50 +02:00
Niko Storni c79e07c9fa change recent livestreams logic slightly 2022-08-10 21:55:29 +02:00
Niko Storni e454cdb4c9 fix post live detection
prevent unlisted videos from ever publishing (even if they were public before and we know about them)
fix timestamp on videos
update user agent
2022-08-10 21:26:36 +02:00
Niko Storni 98a10d1269 fix for channel creation bug 2022-08-10 18:23:37 +02:00
Niko Storni 4f6748ae83 fix bug in channel updates 2022-08-10 17:27:27 +02:00
4 changed files with 144 additions and 185 deletions

No files matched your search

+1 -1
View File
@@ -195,7 +195,7 @@ func getClient(ip *net.TCPAddr) *http.Client {
const (
GoogleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
maxAttempts = 3
extractionError = "YouTube said: Unable to extract video data"
throttledError = "HTTP Error 429"
+100 -170
View File
@@ -1,7 +1,6 @@
package ytdl
import (
"math"
"time"
"github.com/lbryio/ytsync/v5/sdk"
@@ -9,192 +8,123 @@ import (
)
type YtdlVideo struct {
ID string `json:"id"`
Title string `json:"title"`
Formats []struct {
FormatId string `json:"format_id"`
FormatNote string `json:"format_note"`
Ext string `json:"ext"`
Protocol string `json:"protocol"`
Acodec string `json:"acodec"`
Vcodec string `json:"vcodec"`
Url string `json:"url"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Fragments []struct {
Path string `json:"path"`
Duration float64 `json:"duration"`
} `json:"fragments,omitempty"`
AudioExt string `json:"audio_ext"`
VideoExt string `json:"video_ext"`
Format string `json:"format"`
Resolution string `json:"resolution"`
HttpHeaders struct {
UserAgent string `json:"User-Agent"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Asr float64 `json:"asr,omitempty"`
Filesize int64 `json:"filesize,omitempty"`
SourcePreference int `json:"source_preference,omitempty"`
Quality int `json:"quality,omitempty"`
Tbr float64 `json:"tbr,omitempty"`
Language string `json:"language,omitempty"`
LanguagePreference int `json:"language_preference,omitempty"`
Abr float64 `json:"abr,omitempty"`
DownloaderOptions struct {
HttpChunkSize int `json:"http_chunk_size"`
} `json:"downloader_options,omitempty"`
Container string `json:"container,omitempty"`
Fps float64 `json:"fps,omitempty"`
DynamicRange string `json:"dynamic_range,omitempty"`
Vbr float64 `json:"vbr,omitempty"`
FilesizeApprox float64 `json:"filesize_approx,omitempty"`
} `json:"formats"`
Thumbnails []Thumbnail `json:"thumbnails"`
Thumbnail string `json:"thumbnail"`
Description string `json:"description"`
UploadDate string `json:"upload_date"`
uploadDateForReal *time.Time `json:"upload_date_for_real"`
Uploader string `json:"uploader"`
UploaderId string `json:"uploader_id"`
UploaderUrl string `json:"uploader_url"`
ChannelID string `json:"channel_id"`
ChannelUrl string `json:"channel_url"`
Duration int `json:"duration"`
ViewCount int `json:"view_count"`
AgeLimit int `json:"age_limit"`
WebpageUrl string `json:"webpage_url"`
Categories []string `json:"categories"`
Tags []string `json:"tags"`
PlayableInEmbed bool `json:"playable_in_embed"`
IsLive bool `json:"is_live"`
WasLive bool `json:"was_live"`
LiveStatus string `json:"live_status"`
ReleaseTimestamp int64 `json:"release_timestamp"`
LikeCount int `json:"like_count"`
Channel string `json:"channel"`
Availability string `json:"availability"`
WebpageUrlBasename string `json:"webpage_url_basename"`
WebpageUrlDomain string `json:"webpage_url_domain"`
Extractor string `json:"extractor"`
ExtractorKey string `json:"extractor_key"`
DisplayId string `json:"display_id"`
DurationString string `json:"duration_string"`
ReleaseDate string `json:"release_date"`
Asr float64 `json:"asr"`
FormatId string `json:"format_id"`
FormatNote string `json:"format_note"`
SourcePreference int `json:"source_preference"`
Fps float64 `json:"fps"`
Height int `json:"height"`
Quality int `json:"quality"`
Tbr float64 `json:"tbr"`
Url string `json:"url"`
Width int `json:"width"`
Language string `json:"language"`
LanguagePreference int `json:"language_preference"`
Ext string `json:"ext"`
Vcodec string `json:"vcodec"`
Acodec string `json:"acodec"`
DynamicRange string `json:"dynamic_range"`
Protocol string `json:"protocol"`
VideoExt string `json:"video_ext"`
AudioExt string `json:"audio_ext"`
Vbr float64 `json:"vbr"`
Abr float64 `json:"abr"`
Format string `json:"format"`
Resolution string `json:"resolution"`
FilesizeApprox float64 `json:"filesize_approx"`
HttpHeaders struct {
UserAgent string `json:"User-Agent"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Fulltitle string `json:"fulltitle"`
Epoch int `json:"epoch"`
}
ID string `json:"id"`
Title string `json:"title"`
Thumbnails []Thumbnail `json:"thumbnails"`
Description string `json:"description"`
ChannelID string `json:"channel_id"`
Duration int `json:"duration"`
Categories []string `json:"categories"`
Tags []string `json:"tags"`
IsLive bool `json:"is_live"`
LiveStatus string `json:"live_status"`
ReleaseTimestamp int64 `json:"release_timestamp"`
uploadDateForReal *time.Time
Availability string `json:"availability"`
ReleaseDate string `json:"release_date"`
type Format struct {
Asr int `json:"asr"`
Filesize int `json:"filesize"`
FormatID string `json:"format_id"`
FormatNote string `json:"format_note"`
Fps interface{} `json:"fps"`
Height interface{} `json:"height"`
Quality int `json:"quality"`
Tbr float64 `json:"tbr"`
URL string `json:"url"`
Width interface{} `json:"width"`
Ext string `json:"ext"`
Vcodec string `json:"vcodec"`
Acodec string `json:"acodec"`
Abr float64 `json:"abr,omitempty"`
DownloaderOptions struct {
HTTPChunkSize int `json:"http_chunk_size"`
} `json:"downloader_options,omitempty"`
Container string `json:"container,omitempty"`
Format string `json:"format"`
Protocol string `json:"protocol"`
HTTPHeaders struct {
UserAgent string `json:"User-Agent"`
AcceptCharset string `json:"Accept-Charset"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Vbr float64 `json:"vbr,omitempty"`
//WasLive bool `json:"was_live"`
//Formats interface{} `json:"formats"`
//Thumbnail string `json:"thumbnail"`
//Uploader string `json:"uploader"`
//UploaderID string `json:"uploader_id"`
//UploaderURL string `json:"uploader_url"`
//ChannelURL string `json:"channel_url"`
//ViewCount int `json:"view_count"`
//AverageRating interface{} `json:"average_rating"`
//AgeLimit int `json:"age_limit"`
//WebpageURL string `json:"webpage_url"`
//PlayableInEmbed bool `json:"playable_in_embed"`
//AutomaticCaptions interface{} `json:"automatic_captions"`
//Subtitles interface{} `json:"subtitles"`
//Chapters interface{} `json:"chapters"`
//LikeCount int `json:"like_count"`
//Channel string `json:"channel"`
//ChannelFollowerCount int `json:"channel_follower_count"`
//UploadDate string `json:"upload_date"`
//OriginalURL string `json:"original_url"`
//WebpageURLBasename string `json:"webpage_url_basename"`
//WebpageURLDomain string `json:"webpage_url_domain"`
//Extractor string `json:"extractor"`
//ExtractorKey string `json:"extractor_key"`
//Playlist interface{} `json:"playlist"`
//PlaylistIndex interface{} `json:"playlist_index"`
//DisplayID string `json:"display_id"`
//Fulltitle string `json:"fulltitle"`
//DurationString string `json:"duration_string"`
//RequestedSubtitles interface{} `json:"requested_subtitles"`
//HasDrm bool `json:"__has_drm"`
//RequestedFormats interface{} `json:"requested_formats"`
//Format string `json:"format"`
//FormatID string `json:"format_id"`
//Ext string `json:"ext"`
//Protocol string `json:"protocol"`
//Language interface{} `json:"language"`
//FormatNote string `json:"format_note"`
//FilesizeApprox int `json:"filesize_approx"`
//Tbr float64 `json:"tbr"`
//Width int `json:"width"`
//Height int `json:"height"`
//Resolution string `json:"resolution"`
//Fps int `json:"fps"`
//DynamicRange string `json:"dynamic_range"`
//Vcodec string `json:"vcodec"`
//Vbr float64 `json:"vbr"`
//StretchedRatio interface{} `json:"stretched_ratio"`
//Acodec string `json:"acodec"`
//Abr float64 `json:"abr"`
//Asr int `json:"asr"`
//Epoch int `json:"epoch"`
//Filename string `json:"filename"`
//Urls string `json:"urls"`
//Type string `json:"_type"`
}
type Thumbnail struct {
URL string `json:"url"`
Preference int `json:"preference"`
ID string `json:"id"`
Height int `json:"height"`
Width int `json:"width"`
Resolution string `json:"resolution"`
}
type HTTPHeaders struct {
AcceptCharset string `json:"Accept-Charset"`
AcceptLanguage string `json:"Accept-Language"`
AcceptEncoding string `json:"Accept-Encoding"`
Accept string `json:"Accept"`
UserAgent string `json:"User-Agent"`
Height int `json:"height,omitempty"`
Width int `json:"width,omitempty"`
Resolution string `json:"resolution,omitempty"`
}
func (v *YtdlVideo) GetUploadTime() time.Time {
//priority list:
// release timestamp from yt
// release timestamp from morty
// release date from yt
if v.uploadDateForReal != nil {
return *v.uploadDateForReal
}
release, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
var ytdlReleaseTimestamp time.Time
if v.ReleaseTimestamp > 0 {
ytdlReleaseTimestamp = time.Unix(v.ReleaseTimestamp, 0).UTC()
}
//get morty timestamp
var mortyReleaseTimestamp time.Time
mortyRelease, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
if err != nil {
logrus.Error(err)
}
ytdlUploadDate, err := time.Parse("20060102", v.UploadDate)
if err != nil {
logrus.Error(err)
}
if v.ReleaseTimestamp != 0 {
ytdlUploadDate = time.Unix(v.ReleaseTimestamp, 0)
}
if release != nil {
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
if err == nil {
hoursDiff := math.Abs(sqlTime.Sub(ytdlUploadDate).Hours())
if hoursDiff > 48 {
logrus.Infof("upload day from APIs differs from the ytdl one by more than 2 days.")
} else {
v.uploadDateForReal = &sqlTime
return sqlTime
}
} else {
} else {
mortyReleaseTimestamp, err = time.ParseInLocation(time.RFC3339, mortyRelease.ReleaseTime, time.UTC)
if err != nil {
logrus.Error(err)
}
}
v.uploadDateForReal = &ytdlUploadDate
return ytdlUploadDate
ytdlReleaseDate, err := time.Parse("20060102", v.ReleaseDate)
if err != nil {
logrus.Error(err)
}
if !ytdlReleaseTimestamp.IsZero() {
v.uploadDateForReal = &ytdlReleaseTimestamp
} else if !mortyReleaseTimestamp.IsZero() {
v.uploadDateForReal = &mortyReleaseTimestamp
} else {
v.uploadDateForReal = &ytdlReleaseDate
}
return *v.uploadDateForReal
}
+36 -11
View File
@@ -375,7 +375,7 @@ func (s *Sync) ensureChannelOwnership() error {
channelUsesOldMetadata := false
if channelToUse != nil {
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.Languages) == 0 && s.DbChannelData.Language == "")
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.GetLanguages()) == 0 && s.DbChannelData.Language != "")
if !channelUsesOldMetadata {
return nil
}
@@ -434,18 +434,13 @@ func (s *Sync) ensureChannelOwnership() error {
if s.DbChannelData.Language != "" {
languages = []string{s.DbChannelData.Language}
}
//we don't have this data without the API
//if channelInfo.DefaultLanguage != "" {
// if channelInfo.DefaultLanguage == "iw" {
// channelInfo.DefaultLanguage = "he"
// }
// languages = []string{channelInfo.DefaultLanguage}
//}
var locations []jsonrpc.Location = nil
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
}
var c *jsonrpc.TransactionSummary
var recoveredChannelClaimID string
claimCreateOptions := jsonrpc.ClaimCreateOptions{
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description,
@@ -474,16 +469,46 @@ func (s *Sync) ensureChannelOwnership() error {
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
})
if err != nil {
claimId, err2 := s.getChannelClaimIDForTimedOutCreation()
if err2 != nil {
err = errors.Prefix(err2.Error(), err)
} else {
recoveredChannelClaimID = claimId
}
}
}
if err != nil {
return err
}
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
if recoveredChannelClaimID != "" {
s.DbChannelData.ChannelClaimID = recoveredChannelClaimID
} else {
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
}
return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
}
//getChannelClaimIDForTimedOutCreation is a raw function that returns the only channel that exists in the wallet
// this is used because the SDK sucks and can't figure out when to return when creating a claim...
func (s *Sync) getChannelClaimIDForTimedOutCreation() (string, error) {
channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
if err != nil {
return "", err
} else if channels == nil {
return "", errors.Err("no channel response")
}
if len((*channels).Items) != 1 {
return "", errors.Err("more than one channel found when trying to recover from SDK failure in creating the channel")
}
desiredChannel := (*channels).Items[0]
if desiredChannel.Name != s.DbChannelData.DesiredChannelName {
return "", errors.Err("the channel found in the wallet has a different name than the one we expected")
}
return desiredChannel.ClaimID, nil
}
func (s *Sync) addCredits(amountToAdd float64) error {
start := time.Now()
defer func(start time.Time) {
+7 -3
View File
@@ -787,6 +787,9 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
if v.youtubeInfo.IsLive == true {
return nil, errors.Err("video is a live stream and hasn't completed yet")
}
if v.youtubeInfo.Availability != "public" {
return nil, errors.Err("video is not public")
}
if dur > v.maxVideoLength {
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
return nil, errors.Err("video is too long to process")
@@ -795,9 +798,10 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
return nil, errors.Err("video is too short to process")
}
buggedLivestream := v.youtubeInfo.WasLive && time.Unix(v.youtubeInfo.ReleaseTimestamp, 0).After(time.Now().AddDate(0, 0, -1)) && int(dur.Seconds())%7200 == 0
if buggedLivestream {
return nil, errors.Err("livestream is likely bugged as it was published less than 24 hours ago and has a length of %s", dur.String())
buggedLivestream := v.youtubeInfo.LiveStatus == "post_live"
if buggedLivestream && dur >= 2*time.Hour {
return nil, errors.Err("livestream is likely bugged as it was recently published and has a length of %s which is more than 2 hours", dur.String())
}
for {
err = v.download()