Compare commits

..
38 Commits
Author SHA1 Message Date
Niko Storni 618e0843ca update go/deps 2026-01-16 03:07:27 +01:00
Niko Storni 195d9a392d Run gci to sort imports 2026-01-16 03:01:04 +01:00
Andriy Biletsky 1782e8aca4 Remove problematic testdata file (the name does not work with go packager) 2026-01-16 02:59:44 +01:00
Andriy Biletsky 9adccdbae2 Fix and improve incorrect filename tests 2026-01-16 02:59:44 +01:00
Andriy Biletsky c3d27ac06b Implement a more generic encoding method 2026-01-16 02:59:44 +01:00
Alex Grintsvayg 67586dd120 update lbryio/lbcd to include latest bugfix 2026-01-16 02:57:25 +01:00
Alex Grintsvayg 0d3cb2309c minor fix for address test 2026-01-16 02:57:25 +01:00
Alex Grintsvayg a6d11497a2 add stream_name and suggested_file_name to publish 2026-01-16 02:57:25 +01:00
Alex Grintsvayg bb025583e7 fix how errors are returned 2026-01-16 02:57:03 +01:00
Alex Grintsvayg 70f13f27d2 update to latest lbcd 2026-01-16 02:57:03 +01:00
Alex Grintsvayg ebd05aa3f5 uncommitted stuff 2026-01-16 02:57:03 +01:00
Alex Grintsvayg 8f0958bfb2 slimmed down to just the essentials, added publish 2026-01-16 02:57:03 +01:00
Niko Storni cb83b2be3b sort imports 2026-01-16 02:44:22 +01:00
Niko Storni ade76e9f24 remove travis and update go+deps 2026-01-16 02:42:02 +01:00
Niko Storni 7ea3bf01f5 Merge branch 'anbsky-master' 2026-01-16 02:04:20 +01:00
Niko Storni d0c07b4375 fix deprecated lib usage 2026-01-16 02:04:03 +01:00
Andriy Biletsky cf54ec3440 Implement a more generic encoding method 2023-07-12 01:10:11 +07:00
Andriy Biletsky 18c881f040 Add WriteOut method for saving encoded streams straight to files 2023-07-11 21:31:48 +07:00
Niko Storni a01aa6dc06 upgrade dependencies 2023-03-07 19:14:31 +01:00
Niko Storni e6a3f40029 Merge branch 'scheduled-unlisted' 2022-11-01 22:28:32 +01:00
Niko Storni ced09b22ca convert type to variadic 2022-11-01 22:28:18 +01:00
Thomas Zarebczan fa55e82bc1 feat: add scheduled and unlisted 2022-11-01 16:59:07 -04:00
Niko Storni 77944ba3af fix nil ptr 2022-09-27 22:56:34 +02:00
Niko Storni 5f52a995a7 retry failed messages 2022-09-27 22:48:51 +02:00
Niko Storni 014adbb315 actually use the new client 2022-09-27 21:35:13 +02:00
Niko Storni 73228d1bfb use custom http client for slack messages 2022-09-27 20:21:02 +02:00
Niko Storni 69cfd7f798 add account_send
adjust transaction summary fields
2022-09-21 22:53:02 +02:00
Niko Storni 41555cbda2 add helper for special claim tags 2022-09-16 03:18:01 +02:00
Niko Storni 2adb8af5b6 add funding accounts to channel create options 2022-08-15 22:41:00 +02:00
Mark Beamer Jr 9130630afe Added additional functionality to allow for both objects and arrays to be returned from internal-apis client.
Also added a raw API url call and converted current call to a call to a resource so we are not restricted to that format to use the library.
2022-08-05 13:51:05 -04:00
Alex Grin 8e6d493fbf Merge pull request #93 from andybeletsky/master
Add metadata to error returned by jsonrpc
2022-06-15 11:11:14 -04:00
Andrey Beletsky e19facdded Add metadata to error returned by jsonrpc 2022-06-15 00:49:46 +07:00
Niko Storni 365d23f0e2 add support for deterministic pub keys
fix a couple of bugs
2022-06-10 18:18:26 +02:00
Niko Storni e5ab0f883e update dependencies and go 2022-05-04 18:27:35 +02:00
Alex Grin d0aeb0c22b Merge pull request #91 from lbryio/bugfix/jeffreypicard/handle_colons_correctly 2022-03-21 14:25:39 -04:00
Jeffrey Picard 306db74279 We no longer use colons for sequence numbers in urls 2022-03-18 16:27:02 -04:00
Mark Beamer Jr a0391bec79 Extend claim search for use in livestreaming 2022-02-08 16:00:38 -05:00
Alex Grin 5d62502bde Update readme.md 2022-01-17 09:40:58 -05:00
30 changed files with 282 additions and 939 deletions

No files matched your search

-29
View File
@@ -1,29 +0,0 @@
os: linux
dist: xenial
language: go
go:
- 1.15.x
env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "P4YMfllsq/guf5swmBl80IHGvNfUM+5X2b+0LvQQxGJpY9TD6d+PW6U1C90HIT0CDk6UZbtlHyGN+oo+shsWnwMIaM4qmbGdRgtG4TZolXY1QtYQZFW9fTWeHM0wnJeXLV8V4vpTRHnkLrvT8PctbGp5pVOEtcV4y3sEnMP1rkxnMPeHtrDbTlpBmBYEziByEpbg3pe3CE9xVaDD9DqgW7VOOZnQQl7exTtjsIeJYI7OYvdidf/04p632/8WZP2EJbmA2HunfTydWtcD51lCF8V3IvhKKqH4/7sNOKfmUwTDyhLltk9eDcFsYR/FYsy8njH4QDBBrKo/bPq3jzgKF9BY7g28/jMTDY4vDY0Na+4a3+7sDqwdsZ+eiZrgWYUbZ4MwYtpUtcvp8gUz7Avs3v7BBgYsYpgLEW47bT6uNLGO+SdfSwBCYuxT5P0IGCv6wkgYcYgfqVafk+9FiwJlayP2j3wPOdauiYh4WnYX1Rt0zNvaiP5iBhkXwIv3VvUcI1Yu9k0eLDibzZlpR8fkGw3USl+omdjCvXDfJQW1ghMONTm0d7VPG84P/MRhJ+thoh1UwMaT3S0sdPznRnNL6XDntCRT1CZW4l0jfyCPDBW2qw8dWunULYsQZTPWnv4s+BuRRDsfGjyamH5b8evh0TOF+XNOFewmNvflWBNG2y0="
install: true
script:
# Fail if a .go file hasn't been formatted with gofmt
- test -z $(gofmt -s -l $(find . -iname '*.go' -type f))
- make
notifications:
email: false
deploy:
provider: script
skip_cleanup: true
script: ./scripts/deploy.sh
file: bin/lbry
on:
repo: lbryio/lbry.go
tags: true
+2 -1
View File
@@ -3,9 +3,10 @@ package claim
import (
"encoding/hex"
"github.com/cockroachdb/errors"
"github.com/lbryio/lbcd/txscript"
"github.com/lbryio/lbcutil"
"github.com/cockroachdb/errors"
)
func ClaimSupportPayoutScript(name, claimid string, address lbcutil.Address) ([]byte, error) {
+9 -11
View File
@@ -8,7 +8,6 @@ import (
"strings"
"github.com/cockroachdb/errors"
"github.com/lyoshenka/bencode"
)
@@ -67,10 +66,9 @@ func (b Bitmap) Big() *big.Int {
// Cmp compares b and other and returns:
//
// -1 if b < other
// 0 if b == other
// +1 if b > other
//
// -1 if b < other
// 0 if b == other
// +1 if b > other
func (b Bitmap) Cmp(other Bitmap) int {
for k := range b {
if b[k] < other[k] {
@@ -282,13 +280,13 @@ func FromBytesP(data []byte) Bitmap {
return bmp
}
//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
// FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters
func FromString(data string) (Bitmap, error) {
return FromBytes([]byte(data))
}
//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
// FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters otherwise it wil panic.
func FromStringP(data string) Bitmap {
bmp, err := FromString(data)
@@ -298,7 +296,7 @@ func FromStringP(data string) Bitmap {
return bmp
}
//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
// FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters
func FromHex(hexStr string) (Bitmap, error) {
decoded, err := hex.DecodeString(hexStr)
@@ -308,7 +306,7 @@ func FromHex(hexStr string) (Bitmap, error) {
return FromBytes(decoded)
}
//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
// FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters otherwise it wil panic.
func FromHexP(hexStr string) Bitmap {
bmp, err := FromHex(hexStr)
@@ -318,13 +316,13 @@ func FromHexP(hexStr string) Bitmap {
return bmp
}
//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
// FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
func FromShortHex(hexStr string) (Bitmap, error) {
return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr)
}
//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
// FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
// otherwise it wil panic.
func FromShortHexP(hexStr string) Bitmap {
+1 -2
View File
@@ -7,11 +7,10 @@ import (
"sync"
"time"
"golang.org/x/time/rate"
"github.com/lbryio/lbry.go/v3/dht/bits"
"github.com/cockroachdb/errors"
"golang.org/x/time/rate"
)
type queueEdit struct {
+4 -3
View File
@@ -7,10 +7,11 @@ import (
"sync"
"time"
"github.com/cockroachdb/errors"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/lbry.go/v3/dht/bits"
"github.com/lbryio/lbry.go/v3/extras/stop"
"github.com/cockroachdb/errors"
"github.com/davecgh/go-spew/spew"
"github.com/lyoshenka/bencode"
)
@@ -467,7 +468,7 @@ func (n *Node) Store(hash bits.Bitmap, c Contact) {
n.store.Upsert(hash, c)
}
//AddKnownNode adds a known-good node to the routing table
// AddKnownNode adds a known-good node to the routing table
func (n *Node) AddKnownNode(c Contact) {
n.rt.Update(c)
}
+1
View File
@@ -6,6 +6,7 @@ import (
"time"
"github.com/lbryio/lbry.go/v3/dht/bits"
"github.com/lyoshenka/bencode"
)
+2 -1
View File
@@ -6,11 +6,12 @@ import (
"github.com/lbryio/lbry.go/v3/lbrycrd"
"github.com/lbryio/lbry.go/v3/schema/stake"
"github.com/lbryio/lbcutil"
types "github.com/lbryio/types/v2/go"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/lbryio/lbcutil"
"github.com/spf13/cast"
)
+8 -6
View File
@@ -1,7 +1,9 @@
/*Package stop implements the stopper pattern for golang concurrency management. The main use case is to gracefully
exit an application. The pattern allows for a hierarchy of stoppers. Each package should have its own unexported stopper.
The package should maintain startup and shutdown exported methods. If the stopper should stop when another stopper for
a different package stops, the parent argument for the initialization of a stopper be used to create the dependency.
/*
Package stop implements the stopper pattern for golang concurrency management. The main use case is to gracefully
exit an application. The pattern allows for a hierarchy of stoppers. Each package should have its own unexported stopper.
The package should maintain startup and shutdown exported methods. If the stopper should stop when another stopper for
a different package stops, the parent argument for the initialization of a stopper be used to create the dependency.
The package also comes with a debugging tool to help in determining why and where a stopper is not stopping as expected.
If a more complex concurrency is used, it is recommended to implement the library using `DoneNamed` and `AddNamed`.
@@ -75,7 +77,7 @@ func (s *Group) Child() *Group {
return New(s)
}
//AddNamed is the same as Add but will register the functional name of the routine for later output. See `DoneNamed`.
// AddNamed is the same as Add but will register the functional name of the routine for later output. See `DoneNamed`.
func (s *Group) AddNamed(delta int, name string) {
s.Add(delta)
@@ -95,7 +97,7 @@ func (s *Group) AddNamed(delta int, name string) {
}
}
//DoneNamed is the same as `Done` but will output the functional name of all remaining named routines and the waiting on count.
// DoneNamed is the same as `Done` but will output the functional name of all remaining named routines and the waiting on count.
func (s *Group) DoneNamed(name string) {
defer s.Done()
+36 -17
View File
@@ -1,28 +1,47 @@
go 1.25
module github.com/lbryio/lbry.go/v3
require (
github.com/cockroachdb/errors v1.9.0
github.com/cockroachdb/errors v1.12.0
github.com/davecgh/go-spew v1.1.1
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-ini/ini v1.48.0
github.com/golang/protobuf v1.5.2
github.com/gorilla/mux v1.7.3
github.com/gorilla/rpc v1.2.0
github.com/golang/protobuf v1.5.4
github.com/gorilla/mux v1.8.1
github.com/gorilla/rpc v1.2.1
github.com/lbryio/lbcd v0.22.117
github.com/lbryio/lbcutil v1.0.202
github.com/lbryio/lbry.go/v2 v2.7.1
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77
github.com/sergi/go-diff v1.1.0
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cast v1.3.0
github.com/stretchr/testify v1.7.1
go.uber.org/atomic v1.7.0
golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
github.com/sebdah/goldie v1.0.0
github.com/sergi/go-diff v1.4.0
github.com/sirupsen/logrus v1.9.4
github.com/spf13/cast v1.10.0
github.com/stretchr/testify v1.11.1
go.uber.org/atomic v1.11.0
golang.org/x/crypto v0.47.0
golang.org/x/time v0.14.0
gopkg.in/ini.v1 v1.67.1
gotest.tools v2.2.0+incompatible
)
go 1.16
require (
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+59 -832
View File
File diff suppressed because it is too large. Load diff
+1 -2
View File
@@ -3,13 +3,12 @@ package lbrycrd
import (
"encoding/hex"
"golang.org/x/crypto/ripemd160"
"github.com/lbryio/lbcd/chaincfg"
"github.com/lbryio/lbcutil"
"github.com/lbryio/lbcutil/base58"
"github.com/cockroachdb/errors"
"golang.org/x/crypto/ripemd160"
)
// DecodeAddress decodes the string encoding of an address and returns
+3 -2
View File
@@ -4,11 +4,12 @@ import (
"encoding/hex"
c "github.com/lbryio/lbry.go/v3/schema/stake"
"github.com/lbryio/lbcd/btcec"
"github.com/lbryio/lbcd/wire"
pb "github.com/lbryio/types/v2/go"
"github.com/cockroachdb/errors"
"github.com/lbryio/lbcd/btcec"
"github.com/lbryio/lbcd/wire"
)
func NewImageStreamClaim() (*c.Helper, error) {
+1 -1
View File
@@ -17,7 +17,7 @@ import (
"github.com/lbryio/lbcutil"
"github.com/cockroachdb/errors"
"github.com/go-ini/ini"
"gopkg.in/ini.v1"
)
const DefaultPort = 9245
+2 -1
View File
@@ -1,8 +1,9 @@
package lbrycrd
import (
"github.com/cockroachdb/errors"
"github.com/lbryio/lbcd/btcjson"
"github.com/cockroachdb/errors"
)
type outputFinder struct {
+2 -2
View File
@@ -9,12 +9,12 @@ import (
"github.com/lbryio/lbry.go/v3/lbrycrd"
"github.com/lbryio/lbry.go/v3/stream"
pb "github.com/lbryio/types/v2/go"
"github.com/lbryio/lbcd/btcjson"
"github.com/lbryio/lbcd/chaincfg/chainhash"
"github.com/lbryio/lbcd/wire"
"github.com/lbryio/lbcutil"
pb "github.com/lbryio/types/v2/go"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
@@ -108,7 +108,7 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
return st, signedTx, txid, nil
}
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress lbcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
+3 -3
View File
@@ -2,9 +2,9 @@
lbry.go is a set of tools and projects implemented in Golang. See each subfolder for more details
[![Build Status](https://travis-ci.org/lbryio/lbry.go.svg?branch=master)](https://travis-ci.org/lbryio/lbry.go)
**there are significant updates in the [v3 branch](https://github.com/lbryio/lbry.go/tree/v3). if you're starting a new project, strongly consider using that version instead**
This project uses Go modules. Make sure you have Go 1.11+ installed.
This project uses Go modules. Make sure you have Go 1.25+ installed.
- Ubuntu: https://launchpad.net/~longsleep/+archive/ubuntu/golang-backports or https://github.com/golang/go/wiki/Ubuntu
- OSX: `brew install go`
@@ -24,7 +24,7 @@ Make sure you `go fmt` your code before submitting PRs.
This codebase follows the semver standards. You can find the releases [here](https://github.com/lbryio/lbry.go/releases)
You can include the libraries by running `go get github.com/lbryio/lbry.go@latest`.
You can include the libraries by running `go get github.com/lbryio/lbry.go/v2@master`.
You can lock (and you should) to a version whenever you make use of this set of libraries.
## License
-9
View File
@@ -1,9 +0,0 @@
os: linux
dist: trusty
language: go
go:
- 1.15.x
script:
- ./build_and_test.sh
+2 -1
View File
@@ -1,8 +1,9 @@
package address
import (
"github.com/cockroachdb/errors"
"github.com/lbryio/lbry.go/v3/schema/address/base58"
"github.com/cockroachdb/errors"
)
func DecodeAddress(address string, blockchainName string) ([addressLength]byte, error) {
+2 -1
View File
@@ -1,8 +1,9 @@
package address
import (
"github.com/cockroachdb/errors"
"github.com/lbryio/lbry.go/v3/schema/address/base58"
"github.com/cockroachdb/errors"
)
const lbrycrdMainPubkeyPrefix = byte(85)
+4 -3
View File
@@ -6,8 +6,9 @@ import (
"encoding/asn1"
"encoding/pem"
"github.com/cockroachdb/errors"
"github.com/lbryio/lbcd/btcec"
"github.com/cockroachdb/errors"
)
type publicKeyInfo struct {
@@ -41,7 +42,7 @@ func PublicKeyToDER(publicKey *btcec.PublicKey) ([]byte, error) {
}
//This type provides compatibility with the btcec package
// This type provides compatibility with the btcec package
type ecPrivateKey struct {
Version int
PrivateKey []byte
@@ -80,7 +81,7 @@ func GetPrivateKeyFromBytes(privKeyBytes []byte) (*btcec.PrivateKey, *btcec.Publ
return priv, publ, nil
}
//ExtractKeyFromPem returns a btec.Private key object if provided a correct secp256k1 encoded pem.
// ExtractKeyFromPem returns a btec.Private key object if provided a correct secp256k1 encoded pem.
func ExtractKeyFromPem(pm string) (*btcec.PrivateKey, *btcec.PublicKey) {
byta := []byte(pm)
blck, _ := pem.Decode(byta)
+1
View File
@@ -7,6 +7,7 @@ import (
"testing"
"github.com/lbryio/lbcd/btcec"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
+2 -1
View File
@@ -4,8 +4,9 @@ import (
"encoding/hex"
"testing"
"github.com/lbryio/lbcd/btcec"
"github.com/lbryio/lbry.go/v3/schema/keys"
"github.com/lbryio/lbcd/btcec"
pb "github.com/lbryio/types/v2/go"
)
+2 -2
View File
@@ -3,11 +3,11 @@ package stake
import (
"encoding/hex"
"github.com/cockroachdb/errors"
"github.com/lbryio/lbcutil/base58"
v1pb "github.com/lbryio/types/v1/go"
pb "github.com/lbryio/types/v2/go"
"github.com/lbryio/lbcutil/base58"
"github.com/cockroachdb/errors"
)
const lbrySDHash = "lbry_sd_hash"
+1
View File
@@ -9,6 +9,7 @@ import (
"github.com/lbryio/lbry.go/v3/schema/address"
"github.com/lbryio/lbcutil/base58"
"gotest.tools/assert"
)
+2 -1
View File
@@ -4,8 +4,9 @@ import (
"encoding/hex"
"testing"
"github.com/lbryio/lbcd/btcec"
"github.com/lbryio/lbry.go/v3/schema/keys"
"github.com/lbryio/lbcd/btcec"
pb "github.com/lbryio/types/v2/go"
"gotest.tools/assert"
+2 -1
View File
@@ -6,12 +6,13 @@ import (
"github.com/lbryio/lbry.go/v3/schema/address"
"github.com/lbryio/lbry.go/v3/schema/keys"
"github.com/lbryio/lbcd/btcec"
v1PB "github.com/lbryio/types/v1/go"
v2PB "github.com/lbryio/types/v2/go"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/lbryio/lbcd/btcec"
)
type version byte
+2 -1
View File
@@ -7,8 +7,9 @@ import (
"encoding/hex"
"math/big"
"github.com/cockroachdb/errors"
"github.com/lbryio/lbry.go/v2/schema/address"
"github.com/cockroachdb/errors"
)
const SECP256k1 = "SECP256k1"
+39
View File
@@ -3,6 +3,7 @@ package stream
import (
"bytes"
"crypto/sha512"
"fmt"
"hash"
"io"
"math"
@@ -150,6 +151,12 @@ func NewEncoderFromFile(file *os.File) *Encoder {
return e
}
// SetFilename saves provided source filename into stream metadata
func (e *Encoder) SetFilename(filename string) {
e.sd.StreamName = filename
e.sd.SuggestedFileName = sanitizeFilename(filename)
}
// WithIVs sets preset cryptographic material for encoding
func (e *Encoder) WithIVs(key []byte, ivs [][]byte) *Encoder {
e.sd.Key = key
@@ -188,6 +195,7 @@ func (e *Encoder) Next() (Blob, error) {
}
// Stream creates the whole stream in one call
// TODO: Can be refactored to use Encode method
func (e *Encoder) Stream() (Stream, error) {
s := make(Stream, 1, 1+int(math.Ceil(float64(e.srcSizeHint)/maxBlobDataSize))) // len starts at 1 and cap is +1 to leave room for sd blob
@@ -214,6 +222,37 @@ func (e *Encoder) Stream() (Stream, error) {
return s, nil
}
// Encode splits the source into blobs and feeds them into handler function
func (e *Encoder) Encode(handler func(string, []byte) error) ([]string, error) {
manifest := []string{}
for {
blob, err := e.Next()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, err
}
err = handler(blob.HashHex(), blob)
if err != nil {
return nil, fmt.Errorf("cannot process blob: %w", err)
}
manifest = append(manifest, blob.HashHex())
}
sdb := e.SDBlob().ToBlob()
h := sdb.HashHex()
err := handler(h, sdb)
if err != nil {
return nil, fmt.Errorf("cannot handle SD blob: %w", err)
}
manifest = append([]string{h}, manifest...)
return manifest, nil
}
// SDBlob returns the sd blob so far
func (e *Encoder) SDBlob() *SDBlob {
e.sd.updateStreamHash()
+89 -6
View File
@@ -12,6 +12,8 @@ import (
"testing"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/require"
"gotest.tools/assert"
)
var testdataBlobHashes = []string{
@@ -140,6 +142,78 @@ func TestMakeStream(t *testing.T) {
}
}
func TestEncode(t *testing.T) {
blobsToRead := 3
totalBlobs := blobsToRead + 3
data := make([]byte, ((totalBlobs-1)*maxBlobDataSize)+1000) // last blob is partial
_, err := rand.Read(data)
if err != nil {
t.Fatal(err)
}
buf := bytes.NewBuffer(data)
enc := NewEncoder(buf)
stream := make(Stream, blobsToRead+1) // +1 for sd blob
for i := 1; i < blobsToRead+1; i++ { // start at 1 to skip sd blob
stream[i], err = enc.Next()
if err != nil {
t.Fatal(err)
}
}
sdBlob := enc.SDBlob()
if len(sdBlob.BlobInfos) != blobsToRead {
t.Errorf("expected %d blobs in partial sdblob, got %d", blobsToRead, len(sdBlob.BlobInfos))
}
if enc.SourceLen() != maxBlobDataSize*blobsToRead {
t.Errorf("expected length of %d , got %d", maxBlobDataSize*blobsToRead, enc.SourceLen())
}
// now finish the stream, reusing key and IVs
buf = bytes.NewBuffer(data) // rewind to the beginning of the data
enc = NewEncoderFromSD(buf, sdBlob)
outPath := t.TempDir()
handler := func(h string, b []byte) error {
return os.WriteFile(filepath.Join(outPath, h), b, os.ModePerm)
}
writtenManifest, err := enc.Encode(handler)
if err != nil {
t.Fatal(err)
}
if len(writtenManifest) != totalBlobs+1 { // +1 for the terminating blob at the end
t.Errorf("expected %d blobs in stream, got %d", totalBlobs+1, len(writtenManifest))
}
if enc.SourceLen() != len(data) {
t.Errorf("expected length of %d , got %d", len(data), enc.SourceLen())
}
sdb, err := os.ReadFile(filepath.Join(outPath, writtenManifest[0]))
if err != nil {
t.Fatal(err)
}
osdb := enc.SDBlob().ToBlob()
if !bytes.Equal(osdb, sdb) {
t.Errorf("written sd blob does not match original sd blob")
}
for i := 1; i < len(stream); i++ { // start at 1 to skip sd blob
b, err := os.ReadFile(filepath.Join(outPath, writtenManifest[i]))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(stream[i], b) {
t.Errorf("blob %d of reconstructed stream does not match original stream", i)
}
}
}
func TestEmptyStream(t *testing.T) {
enc := NewEncoder(bytes.NewBuffer(nil))
_, err := enc.Next()
@@ -204,15 +278,24 @@ func TestNew(t *testing.T) {
}
func TestNewEncoderFromFile(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", `new "encoder" from file.whatever`))
if err != nil {
t.Error(err)
return
}
sketchyFile := filepath.Join(t.TempDir(), `new "encoder" from file.whatever...`)
file, err := os.OpenFile(sketchyFile, os.O_RDONLY|os.O_CREATE, 0644)
require.NoError(t, err)
file.Close()
file, err = os.Open(sketchyFile)
require.NoError(t, err)
e := NewEncoderFromFile(f)
e := NewEncoderFromFile(file)
if e.sd.SuggestedFileName != "new encoder from file.whatever" {
t.Error("wrong or missing suggested_file_name in sd blob")
}
}
func TestSetFilename(t *testing.T) {
enc := NewEncoder(bytes.NewBuffer(nil))
enc.SetFilename(`filename "sketchy" string`)
assert.Equal(t, "filename sketchy string", enc.sd.SuggestedFileName)
assert.Equal(t, `filename "sketchy" string`, enc.sd.StreamName)
}
View File
Whitespace-only changes.