Compare commits

...
7 Commits
10 changed files with 51 additions and 43 deletions

No files matched your search

+3 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/sirupsen/logrus"
"github.com/lbryio/ytsync/util"
)
@@ -53,7 +54,8 @@ func reflectBlobs() error {
if running {
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
}
logrus.SetLevel(logrus.InfoLevel)
defer logrus.SetLevel(logrus.DebugLevel)
ex, err := os.Executable()
if err != nil {
return errors.Err(err)
+2 -2
View File
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
#MySQL DSN is required for chainquery to store information.
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
#API MySQL DSN is required for chainquery to expose a SQL query service
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
#API Host and Port is required for the API Server to bind and listen on.
#DEFAULT: "0.0.0.0:6300"
+7 -25
View File
@@ -80,6 +80,9 @@ services:
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
###################
## Internal APIs ##
###################
@@ -96,40 +99,19 @@ services:
environment:
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
- MYSQL_USER=lbry
- MYSQL_PASS=lbry
- MYSQL_DATABASE=lbry
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
######################
## MySQL Chainquery ##
######################
mysqlCQ:
image: mysql/mysql-server:5.7.27
restart: "no"
ports:
- "15600:3306"
expose:
- "3306"
environment:
- MYSQL_ALLOW_EMPTY_PASSWORD=true
- MYSQL_DATABASE=chainquery
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
################
## Chainquery ##
################
chainquery:
image: lbry/chainquery:master
image: lbry/chainquery:v2.0.8
restart: "no"
ports:
- 6300:6300
depends_on:
- lbrycrd
- mysqlCQ
- mysql
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
volumes:
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
+5 -5
View File
@@ -56,10 +56,10 @@ echo "successfully started..."
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
# Create Supports for published claim
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
@@ -75,7 +75,7 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
+3
View File
@@ -0,0 +1,3 @@
CREATE DATABASE IF NOT EXISTS chainquery;
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
FLUSH PRIVILEGES;
+3 -3
View File
@@ -47,13 +47,13 @@ func GetIPPool() (*IPPool, error) {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Time{},
LastUse: time.Now().Add(-5 * time.Minute),
})
ipv6Added = true
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Time{},
LastUse: time.Now().Add(-5 * time.Minute),
})
}
}
@@ -144,7 +144,7 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
defer i.lock.Unlock()
sort.Slice(i.ips, func(j, k int) bool {
return i.ips[j].LastUse.Before(i.ips[j].LastUse)
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
})
if !AllThrottled(i.ips) {
+17 -3
View File
@@ -9,14 +9,28 @@ func TestAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ip, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
t.Log(ip)
pool.ReleaseIP(ip)
ip2, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
if ip == ip2 && len(pool.ips) > 1 {
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
}
t.Log(ip2)
pool.ReleaseIP(ip2)
for range pool.ips {
ip, err := pool.GetIP()
_, err = pool.GetIP()
if err != nil {
t.Fatal(err)
}
t.Log(ip)
}
next, err := pool.nextIP()
if err != nil {
t.Logf("%s", err.Error())
+1 -1
View File
@@ -192,7 +192,7 @@ func transferVideos(s *Sync) error {
var stream *jsonrpc.Claim = nil
for _, c := range streams.Items {
if c.ClaimID != video.ClaimID {
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
continue
}
stream = &c
+7 -2
View File
@@ -675,8 +675,13 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
if err != nil {
return nil, errors.Prefix("cannot list claims", err)
}
return claims.Items, nil
items := make([]jsonrpc.Claim, 0, len(claims.Items))
for _, c := range claims.Items {
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
items = append(items, c)
}
}
return items, nil
}
func (s *Sync) checkIntegrity() error {
+3 -1
View File
@@ -236,8 +236,10 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
time.Sleep(ip_manager.IPCooldownPeriod)
sourceAddress, err = v.pool.GetIP()
if err == nil { //TODO: This is possibly not 100% right, but it works so I'm not touching it...
if err == nil {
break
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
return err
}
}
} else {