Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2bde06e4b9 | ||
|
|
5df6db6e96 | ||
|
|
62dfdc1adb | ||
|
|
fe55304184 | ||
|
|
bb76500f04 |
No files matched your search
@@ -87,7 +87,7 @@ services:
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:master #expand_publish
|
||||
image: lbry/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
@@ -104,7 +104,7 @@ services:
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
image: lbry/chainquery:v2.0.8
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
|
||||
+2
-2
@@ -56,8 +56,8 @@ echo "successfully started..."
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=2')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 35 * time.Second
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
@@ -47,13 +47,13 @@ func GetIPPool() (*IPPool, error) {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Time{},
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Time{},
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -144,7 +144,7 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[j].LastUse)
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
|
||||
@@ -9,14 +9,28 @@ func TestAll(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
ip, err := pool.GetIP()
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
}
|
||||
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
|
||||
+1
-1
@@ -192,7 +192,7 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
|
||||
+17
-6
@@ -467,7 +467,7 @@ var thumbnailHosts = []string{
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
@@ -475,7 +475,12 @@ func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
|
||||
if c.SigningChannel == nil {
|
||||
return false
|
||||
}
|
||||
if c.SigningChannel.ClaimID != expectedChannelID {
|
||||
return false
|
||||
}
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
@@ -489,7 +494,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
abandonedClaims := false
|
||||
videoIDs := make(map[string]jsonrpc.Claim)
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -536,7 +541,7 @@ type ytsyncClaim struct {
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -675,8 +680,13 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
|
||||
return claims.Items, nil
|
||||
items := make([]jsonrpc.Claim, 0, len(claims.Items))
|
||||
for _, c := range claims.Items {
|
||||
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
|
||||
items = append(items, c)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
@@ -862,6 +872,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
|
||||
+13
-14
@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
func (v *YoutubeVideo) download() error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -236,8 +236,10 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = v.pool.GetIP()
|
||||
if err == nil { //TODO: This is possibly not 100% right, but it works so I'm not touching it...
|
||||
if err == nil {
|
||||
break
|
||||
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -279,7 +281,7 @@ runcmd:
|
||||
v.pool.SetThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
@@ -424,21 +426,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
var isThrottled bool
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download(isThrottled)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
|
||||
isThrottled = true
|
||||
err = v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
|
||||
Reference in new issue
Block a user