Compare commits

...
4 Commits
Author SHA1 Message Date
Niko Storni 10a855a8ba fix nil ptr dereference
fix bug in transfers
2019-10-12 09:49:49 +02:00
Niko Storni a9e76149e8 parallelize support abandons
update lbrynet support in e2e test
stop using reflector on regtest!
change e2e channel target for more videos
possibly fix a deadlock
2019-10-11 19:42:03 +02:00
Niko Storni dd4aebdba4 make video transferring async
fix bug in videosLimit
change test subject to have more videos
2019-10-11 01:00:04 +02:00
Niko Storni b409d36de0 remove unnecessary output 2019-10-10 18:17:13 +02:00
9 changed files with 216 additions and 95 deletions

No files matched your search

+2 -1
View File
@@ -2,4 +2,5 @@ bin/
e2e/persist
e2e/supporty/supporty
.env
blobsfiles
blobsfiles
ytsync_docker
+1
View File
@@ -6,6 +6,7 @@ lbryum_servers:
- walletserver:50001
save_blobs: true
save_files: false
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
share_usage_data: false
tcp_port: 3333
udp_port: 4444
+1 -1
View File
@@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
+2 -2
View File
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.42.0
image: lbry/wallet-server:v0.42.2
restart: always
environment:
- DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.42.0
image: lbry/lbrynet:v0.42.2
restart: always
ports:
- "15100:5279"
+12 -10
View File
@@ -53,26 +53,28 @@ echo "successfully started..."
./data_setup.sh
# Execute the sync test!
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimAddress=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
# Create Supports for published claim
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 1.0
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 2.0
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 3.0
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 3.0
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
# Reset status for tranfer test
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
# Trigger transfer api
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
# Execute the transfer test!
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
+1
View File
@@ -6,6 +6,7 @@ lbryum_servers:
- walletserver:50001
save_blobs: true
save_files: false
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
share_usage_data: false
tcp_port: 3333
udp_port: 4444
+192 -77
View File
@@ -2,12 +2,18 @@ package manager
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/sdk"
log "github.com/sirupsen/logrus"
"strconv"
)
func waitConfirmations(s *Sync) error {
@@ -39,6 +45,12 @@ waiting:
return nil
}
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
func abandonSupports(s *Sync) (float64, error) {
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
@@ -54,106 +66,209 @@ func abandonSupports(s *Sync) (float64, error) {
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
}
producerWG := &stop.Group{}
claimIDChan := make(chan string, len(allSupports))
abandonRspChan := make(chan abandonResponse, len(allSupports))
alreadyAbandoned := make(map[string]bool, len(allSupports))
producerWG.Add(1)
go func() {
defer producerWG.Done()
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
continue
}
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
}
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
defer s.daemon.SetRPCTimeout(40 * time.Minute)
for i := 0; i < s.ConcurrentVideos; i++ {
consumerWG.Add(1)
go func() {
defer consumerWG.Done()
for {
claimID, more := <-claimIDChan
if !more {
return
} else {
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
} else {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0,
}
continue
}
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
Amount: 0,
}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: nil,
Amount: outputAmount,
}
continue
}
}
}()
}
producerWG.Wait()
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
totalAbandoned := 0.0
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
continue
}
supportOnTransferredClaim := support.Address == s.clientPublishAddress //todo: probably not needed anymore
if supportOnTransferredClaim {
continue
}
alreadyAbandoned[support.ClaimID] = true
summary, err := s.daemon.SupportAbandon(&support.ClaimID, nil, nil, nil, nil)
if err != nil {
return totalAbandoned, errors.Err(err)
}
if len(summary.Outputs) < 1 {
return totalAbandoned, errors.Err("error abandoning supports: no outputs while abandoning %s", support.ClaimID)
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
return totalAbandoned, errors.Err(err)
}
totalAbandoned += outputAmount
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, support.ClaimID)
totalAbandoned += r.Amount
}
return totalAbandoned, nil
}
type updateInfo struct {
ClaimID string
streamUpdateOptions *jsonrpc.StreamUpdateOptions
videoStatus *sdk.VideoStatus
}
func transferVideos(s *Sync) error {
cleanTransfer := true
for _, video := range s.syncedVideos {
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
//log.Debugf("skipping video: %s", video.VideoID)
continue
}
//Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd?
account, err := s.getDefaultAccount()
if err != nil {
return err
}
streams, err := s.daemon.StreamList(&account)
if err != nil {
return errors.Err(err)
}
var stream *jsonrpc.Claim = nil
for _, c := range *streams {
if c.ClaimID != video.ClaimID {
streamChan := make(chan updateInfo, s.ConcurrentVideos)
account, err := s.getDefaultAccount()
if err != nil {
return err
}
streams, err := s.daemon.StreamList(&account)
if err != nil {
return errors.Err(err)
}
producerWG := &stop.Group{}
producerWG.Add(1)
go func() {
defer producerWG.Done()
for _, video := range s.syncedVideos {
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
continue
}
stream = &c
break
}
if stream == nil {
return nil
}
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
},
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
var stream *jsonrpc.Claim = nil
for _, c := range *streams {
if c.ClaimID != video.ClaimID {
continue
}
stream = &c
break
}
if stream == nil {
return
}
videoStatus := sdk.VideoStatus{
ChannelID: s.YoutubeChannelID,
VideoID: video.VideoID,
ClaimID: video.ClaimID,
ClaimName: video.ClaimName,
Status: VideoStatusPublished,
IsTransferred: util.PtrToBool(true),
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
},
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
videoStatus := sdk.VideoStatus{
ChannelID: s.YoutubeChannelID,
VideoID: video.VideoID,
ClaimID: video.ClaimID,
ClaimName: video.ClaimName,
Status: VideoStatusPublished,
IsTransferred: util.PtrToBool(true),
}
streamChan <- updateInfo{
ClaimID: video.ClaimID,
streamUpdateOptions: &streamUpdateOptions,
videoStatus: &videoStatus,
}
}
}()
result, updateError := s.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
if updateError != nil {
cleanTransfer = false
videoStatus.FailureReason = updateError.Error()
videoStatus.Status = VideoStatusTranferFailed
videoStatus.IsTransferred = util.PtrToBool(false)
} else {
videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
}
log.Infof("TRANSFERRED %t", *videoStatus.IsTransferred)
statusErr := s.APIConfig.MarkVideoStatus(videoStatus)
if statusErr != nil {
return errors.Prefix(statusErr.Error(), updateError)
}
if updateError != nil {
return errors.Err(updateError)
}
consumerWG := &stop.Group{}
for i := 0; i < s.ConcurrentVideos; i++ {
consumerWG.Add(1)
go func(worker int) {
defer consumerWG.Done()
for {
ui, more := <-streamChan
if !more {
return
} else {
err := s.streamUpdate(&ui)
if err != nil {
cleanTransfer = false
}
}
}
}(i)
}
// Todo - Transfer Channel as last step and post back to remote db that channel is transferred.
//Transfer channel
producerWG.Wait()
close(streamChan)
consumerWG.Wait()
if !cleanTransfer {
return errors.Err("A video has failed to transfer for the channel...skipping channel transfer")
}
return nil
}
func (s *Sync) streamUpdate(ui *updateInfo) error {
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
if updateError != nil {
ui.videoStatus.FailureReason = updateError.Error()
ui.videoStatus.Status = VideoStatusTranferFailed
ui.videoStatus.IsTransferred = util.PtrToBool(false)
} else {
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
}
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus)
if statusErr != nil {
return errors.Prefix(statusErr.Error(), updateError)
}
return errors.Err(updateError)
}
func transferChannel(s *Sync) error {
account, err := s.getDefaultAccount()
if err != nil {
+1 -2
View File
@@ -492,7 +492,6 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID)
cl, ok := videoIDs[videoID]
if !ok || cl.ClaimID == c.ClaimID {
videoIDs[videoID] = c
@@ -1095,7 +1094,7 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.videosLimit {
log.Println(v.ID() + " is old: skipping")
return nil
}
+4 -2
View File
@@ -124,12 +124,14 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
endpoint := a.ApiURL + "/yt/channel_cert"
res, _ := http.PostForm(endpoint, url.Values{
res, err := http.PostForm(endpoint, url.Values{
"channel_claim_id": {channelID},
"channel_cert": {certHex},
"auth_token": {a.ApiToken},
})
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)