Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef62242f42 | ||
|
|
4a356e86da | ||
|
|
ac95ca57af |
No files matched your search
+3
-3
@@ -53,7 +53,7 @@ echo "successfully started..."
|
||||
./data_setup.sh
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
@@ -66,12 +66,12 @@ videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for tranfer test
|
||||
# Reset status for transfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
|
||||
@@ -125,6 +125,9 @@ func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
|
||||
@@ -56,6 +56,7 @@ func main() {
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
|
||||
+1
-1
@@ -149,7 +149,6 @@ func (s *SyncManager) Start() error {
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
@@ -219,6 +218,7 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
|
||||
+2
-3
@@ -1007,6 +1007,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
@@ -1029,7 +1030,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
@@ -1050,7 +1050,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
nextPageToken = playlistResponse.NextPageToken
|
||||
if nextPageToken == "" {
|
||||
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1065,7 +1065,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
|
||||
}
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
|
||||
@@ -45,6 +45,7 @@ type SyncFlags struct {
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
|
||||
@@ -209,6 +209,10 @@ func (v *YoutubeVideo) download() error {
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"--user-agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
|
||||
Reference in new issue
Block a user