|
|
|
@@ -4,6 +4,7 @@ import (
|
|
|
|
|
"fmt"
|
|
|
|
|
"io/ioutil"
|
|
|
|
|
"math"
|
|
|
|
|
"net"
|
|
|
|
|
"os"
|
|
|
|
|
"os/exec"
|
|
|
|
|
"path/filepath"
|
|
|
|
@@ -23,8 +24,8 @@ import (
|
|
|
|
|
"github.com/lbryio/ytsync/thumbs"
|
|
|
|
|
|
|
|
|
|
"github.com/ChannelMeter/iso8601duration"
|
|
|
|
|
"github.com/asaskevich/govalidator"
|
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
|
|
|
"github.com/nikooo777/ytdl"
|
|
|
|
|
"github.com/shopspring/decimal"
|
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
|
"google.golang.org/api/youtube/v3"
|
|
|
|
@@ -173,33 +174,38 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
|
|
|
|
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (v *YoutubeVideo) fallbackDownload() error {
|
|
|
|
|
cmd := exec.Command("youtube-dl",
|
|
|
|
|
"--no-progress",
|
|
|
|
|
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
|
|
|
|
|
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
|
|
|
|
|
"--merge-output-format",
|
|
|
|
|
"mp4",
|
|
|
|
|
"https://www.youtube.com/watch?v="+v.ID())
|
|
|
|
|
var ipPool []string
|
|
|
|
|
var IPIndex int
|
|
|
|
|
var ipMutex sync.Mutex
|
|
|
|
|
|
|
|
|
|
log.Printf("Running command and waiting for it to finish...")
|
|
|
|
|
output, err := cmd.CombinedOutput()
|
|
|
|
|
log.Debugln(string(output))
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("Command finished with error: %v", errors.Err(string(output)))
|
|
|
|
|
_ = v.delete()
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
func getNextIP() (string, error) {
|
|
|
|
|
ipMutex.Lock()
|
|
|
|
|
defer ipMutex.Unlock()
|
|
|
|
|
if len(ipPool) < 1 {
|
|
|
|
|
IPIndex = 0
|
|
|
|
|
addrs, err := net.InterfaceAddrs()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return "", errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, address := range addrs {
|
|
|
|
|
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
|
|
|
|
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
|
|
|
|
ipPool = append(ipPool, ipnet.IP.String())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
fi, err := os.Stat(v.getFullPath())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
nextIP := ipPool[IPIndex]
|
|
|
|
|
if IPIndex == len(ipPool)-1 {
|
|
|
|
|
IPIndex = 0
|
|
|
|
|
} else {
|
|
|
|
|
IPIndex++
|
|
|
|
|
}
|
|
|
|
|
videoSize := fi.Size()
|
|
|
|
|
v.size = &videoSize
|
|
|
|
|
return nil
|
|
|
|
|
return nextIP, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (v *YoutubeVideo) download() error {
|
|
|
|
|
func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
|
|
|
|
|
videoPath := v.getFullPath()
|
|
|
|
|
|
|
|
|
|
err := os.Mkdir(v.videoDir(), 0750)
|
|
|
|
@@ -215,71 +221,74 @@ func (v *YoutubeVideo) download() error {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
videoUrl := "https://www.youtube.com/watch?v=" + v.id
|
|
|
|
|
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
|
|
|
|
|
useIPv4Env := os.Getenv("USE_IPV4")
|
|
|
|
|
ytdlArgs := []string{
|
|
|
|
|
"--no-progress",
|
|
|
|
|
"--max-filesize",
|
|
|
|
|
fmt.Sprintf("%dM", v.maxVideoSize),
|
|
|
|
|
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
|
|
|
|
|
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
|
|
|
|
|
"--merge-output-format",
|
|
|
|
|
"mp4",
|
|
|
|
|
}
|
|
|
|
|
useIPv4 := useIPv4Env != ""
|
|
|
|
|
if !useIPv4 || youtubeAntiThrottle {
|
|
|
|
|
sourceAddress, err := getNextIP()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
log.Infof("using IPv6: %s", sourceAddress)
|
|
|
|
|
ytdlArgs = append(ytdlArgs,
|
|
|
|
|
"-6",
|
|
|
|
|
"--source-address",
|
|
|
|
|
sourceAddress,
|
|
|
|
|
)
|
|
|
|
|
} else {
|
|
|
|
|
log.Infoln("using IPv4")
|
|
|
|
|
ytdlArgs = append(ytdlArgs, "-4")
|
|
|
|
|
}
|
|
|
|
|
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
|
|
|
|
cmd := exec.Command("youtube-dl", ytdlArgs...)
|
|
|
|
|
|
|
|
|
|
log.Printf("Running command and waiting for it to finish...")
|
|
|
|
|
|
|
|
|
|
stderr, err := cmd.StderrPipe()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
codec := []string{"H.264"}
|
|
|
|
|
ext := []string{"mp4"}
|
|
|
|
|
|
|
|
|
|
//Filter requires a [] interface{}
|
|
|
|
|
codecFilter := make([]interface{}, len(codec))
|
|
|
|
|
for i, v := range codec {
|
|
|
|
|
codecFilter[i] = v
|
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//Filter requires a [] interface{}
|
|
|
|
|
extFilter := make([]interface{}, len(ext))
|
|
|
|
|
for i, v := range ext {
|
|
|
|
|
extFilter[i] = v
|
|
|
|
|
errorLog, _ := ioutil.ReadAll(stderr)
|
|
|
|
|
outLog, _ := ioutil.ReadAll(stdout)
|
|
|
|
|
|
|
|
|
|
if err = cmd.Wait(); err != nil {
|
|
|
|
|
if strings.Contains(err.Error(), "exit status 1") {
|
|
|
|
|
return errors.Err(string(errorLog))
|
|
|
|
|
}
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
|
|
|
|
|
if len(formats) == 0 {
|
|
|
|
|
return errors.Err("no compatible format available for this video")
|
|
|
|
|
}
|
|
|
|
|
maxRetryAttempts := 5
|
|
|
|
|
isLengthLimitSet := v.maxVideoLength > 0.01
|
|
|
|
|
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
|
|
|
|
|
return errors.Err("video is too long to process")
|
|
|
|
|
}
|
|
|
|
|
log.Debugln(string(outLog))
|
|
|
|
|
|
|
|
|
|
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
|
|
|
|
|
formatIndex := i
|
|
|
|
|
if i == maxRetryAttempts-1 {
|
|
|
|
|
formatIndex = len(formats) - 1
|
|
|
|
|
}
|
|
|
|
|
var downloadedFile *os.File
|
|
|
|
|
downloadedFile, err = os.Create(videoPath)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
err = videoInfo.Download(formats[formatIndex], downloadedFile)
|
|
|
|
|
_ = downloadedFile.Close()
|
|
|
|
|
if err != nil {
|
|
|
|
|
//delete the video and ignore the error
|
|
|
|
|
_ = v.delete()
|
|
|
|
|
return errors.Err(err.Error())
|
|
|
|
|
}
|
|
|
|
|
fi, err := os.Stat(v.getFullPath())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
videoSize := fi.Size()
|
|
|
|
|
v.size = &videoSize
|
|
|
|
|
|
|
|
|
|
isVideoSizeLimitSet := v.maxVideoSize > 0
|
|
|
|
|
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
|
|
|
|
|
//delete the video and ignore the error
|
|
|
|
|
_ = v.delete()
|
|
|
|
|
err = errors.Err("file is too big and there is no other format available")
|
|
|
|
|
} else {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
if string(errorLog) != "" {
|
|
|
|
|
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
|
|
|
|
_ = v.delete()
|
|
|
|
|
return errors.Err(string(errorLog))
|
|
|
|
|
}
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
fi, err := os.Stat(v.getFullPath())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return errors.Err(err)
|
|
|
|
|
}
|
|
|
|
|
videoSize := fi.Size()
|
|
|
|
|
v.size = &videoSize
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (v *YoutubeVideo) videoDir() string {
|
|
|
|
@@ -382,7 +391,7 @@ type SyncParams struct {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
|
|
|
|
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
|
|
|
|
|
v.maxVideoSize = int64(params.MaxVideoSize)
|
|
|
|
|
v.maxVideoLength = params.MaxVideoLength
|
|
|
|
|
v.lbryChannelID = params.ChannelID
|
|
|
|
|
v.walletLock = walletLock
|
|
|
|
@@ -394,13 +403,15 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
|
|
|
|
err := v.download()
|
|
|
|
|
err := v.download(false)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
|
|
|
|
|
fallBackErr := v.fallbackDownload()
|
|
|
|
|
if fallBackErr != nil {
|
|
|
|
|
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
|
|
|
|
|
return nil, errors.Prefix("download error", err) //return original error
|
|
|
|
|
if strings.Contains(err.Error(), "HTTP Error 429") {
|
|
|
|
|
err = v.download(true)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, errors.Prefix("download error", err)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
return nil, errors.Prefix("download error", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
log.Debugln("Downloaded " + v.id)
|
|
|
|
|