Compare commits

...
14 Commits
Author SHA1 Message Date
Niko Storni 62a1a0eed5 update goreleaser 2020-01-11 22:11:44 -05:00
Niko Storni 93f9185c71 fix bug when publishing 2020-01-11 22:01:40 -05:00
Niko Storni ef62242f42 add cookies to youtube-dl
minor fixes
2020-01-03 15:59:05 +01:00
Niko Storni 4a356e86da add quicksync for better quota usage 2019-12-27 18:12:41 +01:00
Niko Storni ac95ca57af halt when quota is exceeded 2019-12-27 04:15:48 +01:00
Niko Storni c28d5a716b send warning to slack when video processing panics 2019-12-27 01:39:27 +01:00
Niko Storni 59471f9c26 reduce sync delay
fix stopper bug
2019-12-27 01:27:29 +01:00
Niko Storni 8028c6621c add temporary workaround to release all IPs
speed up one failure
2019-12-26 17:52:44 +01:00
Niko Storni 8abf6ad255 add debug statement 2019-12-24 23:27:16 +01:00
Niko Storni 1f9a645c57 support create fixes
various bug fixes
2019-12-24 05:00:16 +01:00
Niko Storni a5657aaf46 refactor download
get rid of goto
2019-12-20 18:49:33 +01:00
Niko Storni b6861dae9b don't fail on abandoned videos 2019-12-19 03:13:22 +01:00
Niko Storni cff147bb95 work around missing inputs error 2019-12-18 20:24:50 +01:00
Niko Storni 54d04dcf2c add IP debugging
improve process halting
rename improper language code for hebrew
2019-12-18 18:22:15 +01:00
11 changed files with 220 additions and 113 deletions

No files matched your search

+8 -8
View File
@@ -9,12 +9,13 @@ builds:
- amd64
ldflags:
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
archive:
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
replacements:
linux: Linux
amd64: x86_64
format: zip
archives:
- id: zip
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
replacements:
linux: Linux
amd64: x86_64
format: zip
checksum:
name_template: 'checksums.txt'
snapshot:
@@ -24,5 +25,4 @@ changelog:
filters:
exclude:
- '^docs:'
- '^test:'
- '^test:'
+1 -1
View File
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.48.1
image: lbry/wallet-server:v0.50.1
restart: always
environment:
- DB_DIRECTORY=/database
+3 -3
View File
@@ -53,7 +53,7 @@ echo "successfully started..."
./data_setup.sh
# Execute the sync test!
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
@@ -66,12 +66,12 @@ videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
# Reset status for tranfer test
# Reset status for transfer test
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
# Trigger transfer api
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
# Execute the transfer test!
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
+56 -17
View File
@@ -19,20 +19,22 @@ const unbanTimeout = 3 * time.Hour
var stopper = stop.New()
type IPPool struct {
ips []throttledIP
lock *sync.Mutex
ips []throttledIP
lock *sync.RWMutex
stopGrp *stop.Group
}
type throttledIP struct {
IP string
LastUse time.Time
Throttled bool
InUse bool
IP string
UsedForVideo string
LastUse time.Time
Throttled bool
InUse bool
}
var ipPoolInstance *IPPool
func GetIPPool() (*IPPool, error) {
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
if ipPoolInstance != nil {
return ipPoolInstance, nil
}
@@ -59,9 +61,25 @@ func GetIPPool() (*IPPool, error) {
}
}
ipPoolInstance = &IPPool{
ips: pool,
lock: &sync.Mutex{},
ips: pool,
lock: &sync.RWMutex{},
stopGrp: stopGrp,
}
ticker := time.NewTicker(10 * time.Second)
go func() {
for {
select {
case <-stopGrp.Ch():
return
case <-ticker.C:
ipPoolInstance.lock.RLock()
for _, ip := range ipPoolInstance.ips {
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
}
ipPoolInstance.lock.RUnlock()
}
}
}()
return ipPoolInstance, nil
}
@@ -97,12 +115,26 @@ func (i *IPPool) ReleaseIP(ip string) {
if localIP.IP == ip {
localIP.InUse = false
localIP.LastUse = time.Now()
break
return
}
}
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
}
func (i *IPPool) ReleaseAll() {
i.lock.Lock()
defer i.lock.Unlock()
for j, _ := range i.ips {
if i.ips[j].Throttled {
continue
}
localIP := &i.ips[j]
localIP.InUse = false
localIP.LastUse = time.Now()
}
}
func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
func (i *IPPool) SetThrottled(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
var tIP *throttledIP
@@ -129,7 +161,7 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
tIP.Throttled = false
i.lock.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-stopGrp.Ch():
case <-i.stopGrp.Ch():
unbanTimer.Stop()
}
}(tIP)
@@ -138,8 +170,9 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
var ErrAllInUse = errors.Base("all IPs are in use, try again")
var ErrAllThrottled = errors.Base("all IPs are throttled")
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
var ErrInterruptedByUser = errors.Base("interrupted by user")
func (i *IPPool) nextIP() (*throttledIP, error) {
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
i.lock.Lock()
defer i.lock.Unlock()
@@ -165,18 +198,24 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
return nil, errors.Err(ErrResourceLock)
}
nextIP.InUse = true
nextIP.UsedForVideo = forVideo
return nextIP, nil
}
return nil, errors.Err(ErrAllThrottled)
}
func (i *IPPool) GetIP() (string, error) {
func (i *IPPool) GetIP(forVideo string) (string, error) {
for {
ip, err := i.nextIP()
ip, err := i.nextIP(forVideo)
if err != nil {
if errors.Is(err, ErrAllInUse) {
time.Sleep(5 * time.Second)
continue
select {
case <-i.stopGrp.Ch():
return "", errors.Err(ErrInterruptedByUser)
default:
time.Sleep(5 * time.Second)
continue
}
} else if errors.Is(err, ErrAllThrottled) {
return "throttled", err
}
+1
View File
@@ -56,6 +56,7 @@ func main() {
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
+10 -2
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/lbryio/ytsync/blobs_reflector"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
logUtils "github.com/lbryio/ytsync/util"
@@ -92,6 +93,7 @@ const (
TransferStateNotTouched = iota
TransferStatePending
TransferStateComplete
TransferStateManual
)
func (s *SyncManager) Start() error {
@@ -147,7 +149,6 @@ func (s *SyncManager) Start() error {
shouldInterruptLoop = true
} else {
var queuesToSync []string
//TODO: implement scrambling to avoid starvation of queues
if s.syncStatus != "" {
queuesToSync = append(queuesToSync, s.syncStatus)
} else if s.SyncFlags.SyncUpdate {
@@ -159,7 +160,7 @@ func (s *SyncManager) Start() error {
for _, q := range queuesToSync {
//temporary override for sync-until to give tom the time to review the channels
if q == StatusQueued {
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
}
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
if err != nil {
@@ -201,6 +202,12 @@ func (s *SyncManager) Start() error {
shouldNotCount := false
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
err := sync.FullCycle()
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
ipPool, _ := ip_manager.GetIPPool(sync.grp)
if ipPool != nil {
ipPool.ReleaseAll()
}
if err != nil {
fatalErrors := []string{
"default_wallet already exists",
@@ -211,6 +218,7 @@ func (s *SyncManager) Start() error {
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",
"Daily Limit Exceeded",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
+8 -1
View File
@@ -399,6 +399,9 @@ func (s *Sync) ensureChannelOwnership() error {
var languages []string = nil
if channelInfo.DefaultLanguage != "" {
if channelInfo.DefaultLanguage == "iw" {
channelInfo.DefaultLanguage = "he"
}
languages = []string{channelInfo.DefaultLanguage}
}
var locations []jsonrpc.Location = nil
@@ -445,7 +448,11 @@ func (s *Sync) addCredits(amountToAdd float64) error {
return err
}
addressResp, err := s.daemon.AddressUnused(nil)
defaultAccount, err := s.getDefaultAccount()
if err != nil {
return err
}
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
if err != nil {
return err
} else if addressResp == nil {
+6 -1
View File
@@ -204,7 +204,12 @@ func transferVideos(s *Sync) error {
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
ClaimAddress: &s.clientPublishAddress,
FundingAccountIDs: []string{
account,
},
},
},
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
+46 -12
View File
@@ -246,6 +246,8 @@ func (s *Sync) setExceptions() {
}
}
var stopGroup = stop.New()
func (s *Sync) FullCycle() (e error) {
if os.Getenv("HOME") == "" {
return errors.Err("no $HOME env var found")
@@ -258,7 +260,7 @@ func (s *Sync) FullCycle() (e error) {
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stop.New()
s.grp = stopGroup
s.queue = make(chan video)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
@@ -357,7 +359,18 @@ func (s *Sync) processTransfers() (e error) {
isTip := true
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
if err != nil {
return errors.Err(err)
if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
if err != nil {
return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
}
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
if err != nil {
return errors.Err(err)
}
} else {
return errors.Err(err)
}
}
if len(summary.Outputs) < 1 {
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
@@ -638,8 +651,18 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
if sv.Transferred {
_, ok := allClaimsInfo[vID]
if !ok && sv.Published {
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
idsToRemove = append(idsToRemove, vID)
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
if err != nil {
log.Error(err.Error())
continue
}
if len(searchResponse.Claims) == 0 {
log.Debugf("%s: was transferred but appears abandoned! we should ignore this - claimID: %s", vID, sv.ClaimID)
continue //TODO: we should flag these on the db
} else {
log.Debugf("%s: was transferred adn was then edited! we should ignore this - claimID: %s", vID, sv.ClaimID)
//return count, fixed, 0, errors.Err("%s: isn't our control but is on the database and on the blockchain. wtf is up? ClaimID: %s", vID, sv.ClaimID)
}
}
continue
}
@@ -768,7 +791,7 @@ func (s *Sync) doSync() error {
return err
}
if s.transferState != TransferStateComplete {
if s.transferState < TransferStateComplete {
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
if err != nil {
return errors.Prefix("error getting channel cert", err)
@@ -873,12 +896,17 @@ func (s *Sync) startWorker(workerNum int) {
"Sorry about that",
"This video is not available",
"requested format not available",
"interrupted by user",
"Sign in to confirm your age",
"This video is unavailable",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
} else if tryCount < s.MaxTries {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
if util.SubstringInSlice(err.Error(), []string{
"txn-mempool-conflict",
"too-long-mempool-chain",
}) {
log.Println("waiting for a block before retrying")
err := s.waitForNewBlock()
if err != nil {
@@ -890,6 +918,7 @@ func (s *Sync) startWorker(workerNum int) {
"Not enough funds to cover this transaction",
"failed: Not enough funds",
"Error in daemon: Insufficient funds, please deposit additional LBC",
"Missing inputs",
}) {
log.Println("checking funds and UTXOs before retrying...")
err := s.walletSetup()
@@ -975,10 +1004,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
var videos []video
ipPool, err := ip_manager.GetIPPool()
ipPool, err := ip_manager.GetIPPool(s.grp)
if err != nil {
return err
}
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
nextPageToken := ""
for {
@@ -1001,7 +1031,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
return errors.Err("playlist items not found")
}
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
videoIDs := make([]string, 50)
for i, item := range playlistResponse.Items {
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
@@ -1022,7 +1051,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
log.Infof("Got info for %d videos from youtube API", len(videos))
nextPageToken = playlistResponse.NextPageToken
if nextPageToken == "" {
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
break
}
}
@@ -1037,7 +1066,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
sort.Sort(byPublishedAt(videos))
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
Enqueue:
for _, v := range videos {
@@ -1060,7 +1088,7 @@ Enqueue:
func (s *Sync) processVideo(v video) (err error) {
defer func() {
if p := recover(); p != nil {
log.Printf("stack: %s", debug.Stack())
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
var ok bool
err, ok = p.(error)
if !ok {
@@ -1092,6 +1120,7 @@ func (s *Sync) processVideo(v video) (err error) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"giving up after 0 fragment retries",
"Sign in to confirm your age",
}
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
log.Println(v.ID() + " can't ever be published")
@@ -1115,6 +1144,10 @@ func (s *Sync) processVideo(v video) (err error) {
if err != nil {
return err
}
da, err := s.getDefaultAccount()
if err != nil {
return err
}
sp := sources.SyncParams{
ClaimAddress: s.claimAddress,
Amount: publishAmount,
@@ -1123,6 +1156,7 @@ func (s *Sync) processVideo(v video) (err error) {
Namer: s.namer,
MaxVideoLength: s.Manager.maxVideoLength,
Fee: s.Fee,
DefaultAccount: da,
}
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
+1
View File
@@ -45,6 +45,7 @@ type SyncFlags struct {
RemoveDBUnpublished bool
UpgradeMetadata bool
DisableTransfers bool
QuickSync bool
}
type Fee struct {
+80 -68
View File
@@ -199,7 +199,6 @@ func (v *YoutubeVideo) download() error {
"480",
"320",
}
qualityIndex := 0
ytdlArgs := []string{
"--no-progress",
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
@@ -210,6 +209,10 @@ func (v *YoutubeVideo) download() error {
"--abort-on-unavailable-fragment",
"--fragment-retries",
"0",
"--user-agent",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
"--cookies",
"cookies.txt",
}
if v.maxVideoSize > 0 {
ytdlArgs = append(ytdlArgs,
@@ -224,94 +227,94 @@ func (v *YoutubeVideo) download() error {
)
}
sourceAddress, err := v.pool.GetIP()
if err != nil {
if sourceAddress == "throttled" {
for {
var sourceAddress string
for {
sourceAddress, err = v.pool.GetIP(v.id)
if err != nil {
if errors.Is(err, ip_manager.ErrAllThrottled) {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
default:
time.Sleep(ip_manager.IPCooldownPeriod)
continue
}
time.Sleep(ip_manager.IPCooldownPeriod)
sourceAddress, err = v.pool.GetIP()
if err == nil {
break
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
return err
}
} else {
return err
}
} else {
return errors.Err(err)
}
break
}
defer v.pool.ReleaseIP(sourceAddress)
ytdlArgs = append(ytdlArgs,
"--source-address",
sourceAddress,
"https://www.youtube.com/watch?v="+v.ID(),
)
runcmd:
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
cmd := exec.Command("youtube-dl", argsWithFilters...)
for i, quality := range qualities {
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
cmd := exec.Command("youtube-dl", argsWithFilters...)
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress, v.stopGroup)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
err = cmd.Wait()
if err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
if i == (len(qualities) - 1) {
return errors.Err(string(errorLog))
}
continue //this bypasses the yt throttling IP redistribution... TODO: don't
}
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete("does not pass filter duration")
return errors.Err("video is too long to process")
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete("File is larger than max-filesize")
return errors.Err("the video is too big to sync, skipping for now")
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete("due to error")
return errors.Err(string(errorLog))
}
return errors.Err(err)
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
err = os.Chmod(v.getFullPath(), 0777)
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
break
}
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete("does not pass filter duration")
return errors.Err("video is too long to process")
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete("File is larger than max-filesize")
return errors.Err("the video is too big to sync, skipping for now")
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete("due to error")
return errors.Err(string(errorLog))
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
err = os.Chmod(v.getFullPath(), 0777)
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
@@ -364,7 +367,6 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
languages, locations, tags := v.getMetadata()
var fee *jsonrpc.Fee
if params.Fee != nil {
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
@@ -387,6 +389,9 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
ThumbnailURL: &v.thumbnailURL,
Tags: tags,
Locations: locations,
FundingAccountIDs: []string{
params.DefaultAccount,
},
},
Fee: fee,
License: util.PtrToString("Copyrighted (contact publisher)"),
@@ -412,6 +417,7 @@ type SyncParams struct {
Namer *namer.Namer
MaxVideoLength float64
Fee *sdk.Fee
DefaultAccount string
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
@@ -459,6 +465,9 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
tags = nil
if !v.mocked {
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
v.youtubeInfo.Snippet.DefaultLanguage = "he"
}
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}
@@ -542,6 +551,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
ThumbnailURL: &thumbnailURL,
Languages: languages,
Locations: locations,
FundingAccountIDs: []string{
params.DefaultAccount,
},
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),