Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe55304184 | ||
|
|
bb76500f04 | ||
|
|
4e2efec66e | ||
|
|
8d650e032c | ||
|
|
f5d97957e7 | ||
|
|
36f539ef5d | ||
|
|
77461f71af | ||
|
|
caca13de61 |
No files matched your search
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
)
|
||||
@@ -53,7 +54,8 @@ func reflectBlobs() error {
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
defer logrus.SetLevel(logrus.DebugLevel)
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
|
||||
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
|
||||
#MySQL DSN is required for chainquery to store information.
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API MySQL DSN is required for chainquery to expose a SQL query service
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API Host and Port is required for the API Server to bind and listen on.
|
||||
#DEFAULT: "0.0.0.0:6300"
|
||||
|
||||
+7
-25
@@ -80,6 +80,9 @@ services:
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
###################
|
||||
## Internal APIs ##
|
||||
###################
|
||||
@@ -96,40 +99,19 @@ services:
|
||||
environment:
|
||||
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASS=lbry
|
||||
- MYSQL_DATABASE=lbry
|
||||
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
######################
|
||||
## MySQL Chainquery ##
|
||||
######################
|
||||
mysqlCQ:
|
||||
image: mysql/mysql-server:5.7.27
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15600:3306"
|
||||
expose:
|
||||
- "3306"
|
||||
environment:
|
||||
- MYSQL_ALLOW_EMPTY_PASSWORD=true
|
||||
- MYSQL_DATABASE=chainquery
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
image: lbry/chainquery:v2.0.8
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- mysqlCQ
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
|
||||
+5
-5
@@ -56,10 +56,10 @@ echo "successfully started..."
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
@@ -75,7 +75,7 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE DATABASE IF NOT EXISTS chainquery;
|
||||
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
+155
-86
@@ -1,122 +1,191 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"net"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 25 * time.Second
|
||||
const IPCooldownPeriod = 35 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var ipv6Pool []string
|
||||
var ipv4Pool []string
|
||||
var throttledIPs map[string]bool
|
||||
var ipInUse map[string]bool
|
||||
var ipLastUsed map[string]time.Time
|
||||
var ipMutex sync.Mutex
|
||||
var stopper = stop.New()
|
||||
|
||||
func GetNextIP(ipv6 bool) (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
|
||||
throttledIPs = make(map[string]bool)
|
||||
ipInUse = make(map[string]bool)
|
||||
ipLastUsed = make(map[string]time.Time)
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.Mutex
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
}
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool() (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
ipv6Added := false
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
nextIP := ""
|
||||
if ipv6 {
|
||||
nextIP = getLeastUsedIP(ipv6Pool)
|
||||
} else {
|
||||
nextIP = getLeastUsedIP(ipv4Pool)
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.Mutex{},
|
||||
}
|
||||
if nextIP == "" {
|
||||
return "throttled", errors.Err("all IPs are throttled")
|
||||
}
|
||||
lastUse := ipLastUsed[nextIP]
|
||||
if time.Since(lastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
|
||||
}
|
||||
|
||||
ipInUse[nextIP] = true
|
||||
return nextIP, nil
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
func ReleaseIP(ip string) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
ipLastUsed[ip] = time.Now()
|
||||
ipInUse[ip] = false
|
||||
}
|
||||
|
||||
func getLeastUsedIP(ipPool []string) string {
|
||||
nextIP := ""
|
||||
veryLastUse := time.Now()
|
||||
for _, ip := range ipPool {
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
continue
|
||||
}
|
||||
inUse := ipInUse[ip]
|
||||
if inUse {
|
||||
continue
|
||||
}
|
||||
lastUse := ipLastUsed[ip]
|
||||
if lastUse.Before(veryLastUse) {
|
||||
nextIP = ip
|
||||
veryLastUse = lastUse
|
||||
// AllThrottled checks whether the IPs provided are all throttled.
|
||||
// returns false if at least one IP is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllThrottled(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return nextIP
|
||||
return true
|
||||
}
|
||||
|
||||
func SetIpThrottled(ip string, stopGrp *stop.Group) {
|
||||
ipMutex.Lock()
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
return
|
||||
// AllInUse checks whether the IPs provided are all currently in use.
|
||||
// returns false if at least one IP is not in use AND is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllInUse(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.InUse && !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseIP(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
if localIP.Throttled {
|
||||
return
|
||||
}
|
||||
localIP.Throttled = true
|
||||
tIP = localIP
|
||||
break
|
||||
}
|
||||
}
|
||||
throttledIPs[ip] = true
|
||||
ipMutex.Unlock()
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func() {
|
||||
go func(tIP *throttledIP) {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
ipMutex.Lock()
|
||||
throttledIPs[ip] = false
|
||||
ipMutex.Unlock()
|
||||
i.lock.Lock()
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}()
|
||||
}(tIP)
|
||||
}
|
||||
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
|
||||
func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
if AllInUse(i.ips) {
|
||||
return nil, errors.Err(ErrAllInUse)
|
||||
}
|
||||
|
||||
var nextIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
ip := &i.ips[j]
|
||||
if ip.InUse || ip.Throttled {
|
||||
continue
|
||||
}
|
||||
nextIP = ip
|
||||
break
|
||||
}
|
||||
if nextIP == nil {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP() (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP()
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if time.Since(ip.LastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
|
||||
}
|
||||
return ip.IP, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
pool, err := GetIPPool()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
} else {
|
||||
t.Fatal(next)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -192,7 +192,7 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
|
||||
+14
-4
@@ -14,6 +14,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
@@ -674,8 +675,13 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
|
||||
return claims.Items, nil
|
||||
items := make([]jsonrpc.Claim, 0, len(claims.Items))
|
||||
for _, c := range claims.Items {
|
||||
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
|
||||
items = append(items, c)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
@@ -963,6 +969,10 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
|
||||
var videos []video
|
||||
ipPool, err := ip_manager.GetIPPool()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
@@ -1000,7 +1010,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
@@ -1016,7 +1026,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+17
-23
@@ -50,6 +50,7 @@ type YoutubeVideo struct {
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
pool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
@@ -87,7 +88,7 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
@@ -101,9 +102,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
@@ -112,6 +114,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +223,8 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
)
|
||||
}
|
||||
sourceAddress, err := ip_manager.GetNextIP(useIPv6)
|
||||
|
||||
sourceAddress, err := v.pool.GetIP()
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
@@ -231,8 +235,8 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
}
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = ip_manager.GetNextIP(useIPv6)
|
||||
if err == nil {
|
||||
sourceAddress, err = v.pool.GetIP()
|
||||
if err == nil { //TODO: This is possibly not 100% right, but it works so I'm not touching it...
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -240,23 +244,13 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
defer ip_manager.ReleaseIP(sourceAddress)
|
||||
if useIPv6 {
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-6",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
} else {
|
||||
log.Infof("using IPv4: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-4",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
}
|
||||
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
runcmd:
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
@@ -282,7 +276,7 @@ runcmd:
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
ip_manager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
v.pool.SetThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
|
||||
Reference in new issue
Block a user