Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10a855a8ba | ||
|
|
a9e76149e8 | ||
|
|
dd4aebdba4 | ||
|
|
b409d36de0 |
No files matched your search
+2
-1
@@ -2,4 +2,5 @@ bin/
|
||||
e2e/persist
|
||||
e2e/supporty/supporty
|
||||
.env
|
||||
blobsfiles
|
||||
blobsfiles
|
||||
ytsync_docker
|
||||
@@ -6,6 +6,7 @@ lbryum_servers:
|
||||
- walletserver:50001
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
|
||||
+1
-1
@@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.42.0
|
||||
image: lbry/wallet-server:v0.42.2
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.42.0
|
||||
image: lbry/lbrynet:v0.42.2
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
|
||||
+12
-10
@@ -53,26 +53,28 @@ echo "successfully started..."
|
||||
./data_setup.sh
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimAddress=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID}" "${videoClaimAddress}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for tranfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
|
||||
@@ -6,6 +6,7 @@ lbryum_servers:
|
||||
- walletserver:50001
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
|
||||
+192
-77
@@ -2,12 +2,18 @@ package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
@@ -39,6 +45,12 @@ waiting:
|
||||
return nil
|
||||
}
|
||||
|
||||
type abandonResponse struct {
|
||||
ClaimID string
|
||||
Error error
|
||||
Amount float64
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
@@ -54,106 +66,209 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
|
||||
claimIDChan := make(chan string, len(allSupports))
|
||||
abandonRspChan := make(chan abandonResponse, len(allSupports))
|
||||
alreadyAbandoned := make(map[string]bool, len(allSupports))
|
||||
producerWG.Add(1)
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
claimIDChan <- support.ClaimID
|
||||
}
|
||||
}()
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(30 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(40 * time.Minute)
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: nil,
|
||||
Amount: outputAmount,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
producerWG.Wait()
|
||||
close(claimIDChan)
|
||||
consumerWG.Wait()
|
||||
close(abandonRspChan)
|
||||
|
||||
totalAbandoned := 0.0
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
for r := range abandonRspChan {
|
||||
if r.Error != nil {
|
||||
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
|
||||
continue
|
||||
}
|
||||
supportOnTransferredClaim := support.Address == s.clientPublishAddress //todo: probably not needed anymore
|
||||
if supportOnTransferredClaim {
|
||||
continue
|
||||
}
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
summary, err := s.daemon.SupportAbandon(&support.ClaimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
return totalAbandoned, errors.Err(err)
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return totalAbandoned, errors.Err("error abandoning supports: no outputs while abandoning %s", support.ClaimID)
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
return totalAbandoned, errors.Err(err)
|
||||
}
|
||||
totalAbandoned += outputAmount
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, support.ClaimID)
|
||||
totalAbandoned += r.Amount
|
||||
}
|
||||
return totalAbandoned, nil
|
||||
}
|
||||
|
||||
type updateInfo struct {
|
||||
ClaimID string
|
||||
streamUpdateOptions *jsonrpc.StreamUpdateOptions
|
||||
videoStatus *sdk.VideoStatus
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
cleanTransfer := true
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
|
||||
//log.Debugf("skipping video: %s", video.VideoID)
|
||||
continue
|
||||
}
|
||||
|
||||
//Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd?
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := s.daemon.StreamList(&account)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range *streams {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
streamChan := make(chan updateInfo, s.ConcurrentVideos)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := s.daemon.StreamList(&account)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
producerWG.Add(1)
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
break
|
||||
}
|
||||
if stream == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range *streams {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
break
|
||||
}
|
||||
if stream == nil {
|
||||
return
|
||||
}
|
||||
|
||||
videoStatus := sdk.VideoStatus{
|
||||
ChannelID: s.YoutubeChannelID,
|
||||
VideoID: video.VideoID,
|
||||
ClaimID: video.ClaimID,
|
||||
ClaimName: video.ClaimName,
|
||||
Status: VideoStatusPublished,
|
||||
IsTransferred: util.PtrToBool(true),
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
videoStatus := sdk.VideoStatus{
|
||||
ChannelID: s.YoutubeChannelID,
|
||||
VideoID: video.VideoID,
|
||||
ClaimID: video.ClaimID,
|
||||
ClaimName: video.ClaimName,
|
||||
Status: VideoStatusPublished,
|
||||
IsTransferred: util.PtrToBool(true),
|
||||
}
|
||||
streamChan <- updateInfo{
|
||||
ClaimID: video.ClaimID,
|
||||
streamUpdateOptions: &streamUpdateOptions,
|
||||
videoStatus: &videoStatus,
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
result, updateError := s.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
|
||||
if updateError != nil {
|
||||
cleanTransfer = false
|
||||
videoStatus.FailureReason = updateError.Error()
|
||||
videoStatus.Status = VideoStatusTranferFailed
|
||||
videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", *videoStatus.IsTransferred)
|
||||
statusErr := s.APIConfig.MarkVideoStatus(videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Prefix(statusErr.Error(), updateError)
|
||||
}
|
||||
if updateError != nil {
|
||||
return errors.Err(updateError)
|
||||
}
|
||||
consumerWG := &stop.Group{}
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func(worker int) {
|
||||
defer consumerWG.Done()
|
||||
for {
|
||||
ui, more := <-streamChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
err := s.streamUpdate(&ui)
|
||||
if err != nil {
|
||||
cleanTransfer = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
// Todo - Transfer Channel as last step and post back to remote db that channel is transferred.
|
||||
//Transfer channel
|
||||
producerWG.Wait()
|
||||
close(streamChan)
|
||||
consumerWG.Wait()
|
||||
|
||||
if !cleanTransfer {
|
||||
return errors.Err("A video has failed to transfer for the channel...skipping channel transfer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
|
||||
if updateError != nil {
|
||||
ui.videoStatus.FailureReason = updateError.Error()
|
||||
ui.videoStatus.Status = VideoStatusTranferFailed
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
|
||||
statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Prefix(statusErr.Error(), updateError)
|
||||
}
|
||||
return errors.Err(updateError)
|
||||
}
|
||||
|
||||
func transferChannel(s *Sync) error {
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
|
||||
+1
-2
@@ -492,7 +492,6 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
|
||||
log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID)
|
||||
cl, ok := videoIDs[videoID]
|
||||
if !ok || cl.ClaimID == c.ClaimID {
|
||||
videoIDs[videoID] = c
|
||||
@@ -1095,7 +1094,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
|
||||
if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.videosLimit {
|
||||
log.Println(v.ID() + " is old: skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
+4
-2
@@ -124,12 +124,14 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
|
||||
endpoint := a.ApiURL + "/yt/channel_cert"
|
||||
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"channel_claim_id": {channelID},
|
||||
"channel_cert": {certHex},
|
||||
"auth_token": {a.ApiToken},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
|
||||
Reference in new issue
Block a user