Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
89fe1e99d6 | ||
|
|
ec81af0857 | ||
|
|
51cb7f5e34 | ||
|
|
8ad2acc936 | ||
|
|
b8caf3f82b | ||
|
|
b34d328cba | ||
|
|
a0dcb2216d | ||
|
|
9d6a7e03cd | ||
|
|
e44e272689 |
No files matched your search
@@ -55,7 +55,7 @@ func reflectBlobs() error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exPath := filepath.Dir(ex)
|
||||
config, err := loadConfig(exPath + "/prismconfig.json")
|
||||
config, err := loadConfig(exPath + "/prism_config.json")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -63,6 +63,13 @@ func reflectBlobs() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := dbHandle.CloseDB()
|
||||
if err != nil {
|
||||
log.Errorf("failed to close db handle: %s", err.Error())
|
||||
}
|
||||
|
||||
}()
|
||||
st := store.NewDBBackedS3Store(
|
||||
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
|
||||
dbHandle)
|
||||
|
||||
@@ -5,6 +5,7 @@ require (
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/PuerkitoBio/goquery v1.5.0 // indirect
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
|
||||
github.com/aws/aws-sdk-go v1.17.3
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/go-ini/ini v1.42.0 // indirect
|
||||
@@ -18,7 +19,7 @@ require (
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
|
||||
github.com/lbryio/lbry.go v1.0.15
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710153751-817fa4aa1df5
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
|
||||
|
||||
@@ -25,6 +25,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
|
||||
@@ -197,8 +198,8 @@ github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710153751-817fa4aa1df5 h1:dWnn/y2wiaoD6liTpsjLYhWeOSlmd7lndBtysBkX5ro=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710153751-817fa4aa1df5/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
|
||||
@@ -199,6 +199,7 @@ func (s *SyncManager) Start() error {
|
||||
"failure uploading wallet",
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"HTTP Error 429",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
|
||||
+2
-1
@@ -656,6 +656,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"HTTP Error 429",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
|
||||
s.grp.Stop()
|
||||
@@ -677,7 +678,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"Invalid status code: 403",
|
||||
"HTTP Error 403",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
|
||||
+96
-85
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -23,8 +24,8 @@ import (
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/nikooo777/ytdl"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
@@ -173,33 +174,38 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) fallbackDownload() error {
|
||||
cmd := exec.Command("youtube-dl",
|
||||
"--no-progress",
|
||||
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
|
||||
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"https://www.youtube.com/watch?v="+v.ID())
|
||||
var ipPool []string
|
||||
var IPIndex int
|
||||
var ipMutex sync.Mutex
|
||||
|
||||
log.Printf("Running command and waiting for it to finish...")
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.Debugln(string(output))
|
||||
if err != nil {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(output)))
|
||||
_ = v.delete()
|
||||
return errors.Err(err)
|
||||
func getNextIP() (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipPool) < 1 {
|
||||
IPIndex = 0
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipPool = append(ipPool, ipnet.IP.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
nextIP := ipPool[IPIndex]
|
||||
if IPIndex == len(ipPool)-1 {
|
||||
IPIndex = 0
|
||||
} else {
|
||||
IPIndex++
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
return nextIP, nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0750)
|
||||
@@ -215,71 +221,74 @@ func (v *YoutubeVideo) download() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
videoUrl := "https://www.youtube.com/watch?v=" + v.id
|
||||
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
|
||||
useIPv4Env := os.Getenv("USE_IPV4")
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"--max-filesize",
|
||||
fmt.Sprintf("%dM", v.maxVideoSize),
|
||||
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
|
||||
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
}
|
||||
useIPv4 := useIPv4Env != ""
|
||||
if !useIPv4 || youtubeAntiThrottle {
|
||||
sourceAddress, err := getNextIP()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-6",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
} else {
|
||||
log.Infoln("using IPv4")
|
||||
ytdlArgs = append(ytdlArgs, "-4")
|
||||
}
|
||||
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
||||
cmd := exec.Command("youtube-dl", ytdlArgs...)
|
||||
|
||||
log.Printf("Running command and waiting for it to finish...")
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
codec := []string{"H.264"}
|
||||
ext := []string{"mp4"}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
codecFilter := make([]interface{}, len(codec))
|
||||
for i, v := range codec {
|
||||
codecFilter[i] = v
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
extFilter := make([]interface{}, len(ext))
|
||||
for i, v := range ext {
|
||||
extFilter[i] = v
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
|
||||
if len(formats) == 0 {
|
||||
return errors.Err("no compatible format available for this video")
|
||||
}
|
||||
maxRetryAttempts := 5
|
||||
isLengthLimitSet := v.maxVideoLength > 0.01
|
||||
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
|
||||
formatIndex := i
|
||||
if i == maxRetryAttempts-1 {
|
||||
formatIndex = len(formats) - 1
|
||||
}
|
||||
var downloadedFile *os.File
|
||||
downloadedFile, err = os.Create(videoPath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = videoInfo.Download(formats[formatIndex], downloadedFile)
|
||||
_ = downloadedFile.Close()
|
||||
if err != nil {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
return errors.Err(err.Error())
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
|
||||
isVideoSizeLimitSet := v.maxVideoSize > 0
|
||||
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
err = errors.Err("file is too big and there is no other format available")
|
||||
} else {
|
||||
break
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete()
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) videoDir() string {
|
||||
@@ -382,7 +391,7 @@ type SyncParams struct {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
|
||||
v.maxVideoSize = int64(params.MaxVideoSize)
|
||||
v.maxVideoLength = params.MaxVideoLength
|
||||
v.lbryChannelID = params.ChannelID
|
||||
v.walletLock = walletLock
|
||||
@@ -394,13 +403,15 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download()
|
||||
err := v.download(false)
|
||||
if err != nil {
|
||||
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
|
||||
fallBackErr := v.fallbackDownload()
|
||||
if fallBackErr != nil {
|
||||
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
|
||||
return nil, errors.Prefix("download error", err) //return original error
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
err = v.download(true)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
}
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
Reference in new issue
Block a user