Compare commits
156
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cc69faa1fd | ||
|
|
0f53cd86c8 | ||
|
|
212c8f188d | ||
|
|
dec248adec | ||
|
|
98e6a066f4 | ||
|
|
c90db54a3d | ||
|
|
927d156933 | ||
|
|
c71f91073f | ||
|
|
ab28387692 | ||
|
|
c4905d02b9 | ||
|
|
ee39880fb5 | ||
|
|
926b3e56b9 | ||
|
|
e060df5367 | ||
|
|
a3294d4a0d | ||
|
|
19ce0ab246 | ||
|
|
75a1cc0d33 | ||
|
|
05228529b0 | ||
|
|
3cbeadfbc3 | ||
|
|
3d4938c0e2 | ||
|
|
1b850b8a2b | ||
|
|
ebd33f1869 | ||
|
|
ac846b4df3 | ||
|
|
0d427c9b90 | ||
|
|
dc6194f862 | ||
|
|
de50214e1f | ||
|
|
5ec4a88c35 | ||
|
|
31c141e757 | ||
|
|
9d5760d899 | ||
|
|
7945e1ea3c | ||
|
|
d94c40e371 | ||
|
|
31f22122e8 | ||
|
|
506582aa2b | ||
|
|
a02b251c9b | ||
|
|
34cbb6fa79 | ||
|
|
33679b56fd | ||
|
|
8897c4d560 | ||
|
|
f0c18ccbe7 | ||
|
|
e560d83c51 | ||
|
|
eca7addc67 | ||
|
|
263f3ba5c9 | ||
|
|
f514123ef0 | ||
|
|
9f461db0d0 | ||
|
|
7de8670616 | ||
|
|
8c7908b200 | ||
|
|
e7722e039f | ||
|
|
039bc0208a | ||
|
|
d57900a069 | ||
|
|
c1153302aa | ||
|
|
1086a3297f | ||
|
|
5c613934ca | ||
|
|
6d7e9092f8 | ||
|
|
8bb6f328dc | ||
|
|
7f03b13579 | ||
|
|
09008ea991 | ||
|
|
ee234212e6 | ||
|
|
277b243f52 | ||
|
|
0973ac753f | ||
|
|
f41cfbfb97 | ||
|
|
a70980c81a | ||
|
|
12a962f656 | ||
|
|
5936444f3e | ||
|
|
c7d42f00c6 | ||
|
|
b1318a9958 | ||
|
|
d2560d260c | ||
|
|
f40a61cf9a | ||
|
|
e0623578bf | ||
|
|
ead8daaa14 | ||
|
|
23b4b9e230 | ||
|
|
947017e334 | ||
|
|
7fd0d6507f | ||
|
|
ad1e9ef086 | ||
|
|
2f1d08e417 | ||
|
|
55f4eb80ba | ||
|
|
7c160ff65e | ||
|
|
fc5d5faaed | ||
|
|
5394f1763c | ||
|
|
050b67c9d6 | ||
|
|
4b87cb45ee | ||
|
|
f0e1db319c | ||
|
|
3950715237 | ||
|
|
ae9ba14b59 | ||
|
|
ad6c6fbe35 | ||
|
|
ffa5c20c88 | ||
|
|
5a6218eeca | ||
|
|
200531dd96 | ||
|
|
15a2f048ac | ||
|
|
d317a4042c | ||
|
|
c2d717aba5 | ||
|
|
ef74777df1 | ||
|
|
f6dbf99fb5 | ||
|
|
0b2d9e15b4 | ||
|
|
d14e5e75e8 | ||
|
|
df11ef34d2 | ||
|
|
4fc619262c | ||
|
|
2750eee86d | ||
|
|
71cb03345b | ||
|
|
22c8ca6c40 | ||
|
|
237a8965eb | ||
|
|
7d872c7863 | ||
|
|
0d08858dfb | ||
|
|
5c6d63c93d | ||
|
|
c8c10d2bb0 | ||
|
|
f4d3e9ea6a | ||
|
|
3713d3488d | ||
|
|
9b46d03c91 | ||
|
|
fe4e738580 | ||
|
|
205b0c4263 | ||
|
|
78eef25f1a | ||
|
|
c9305fd070 | ||
|
|
20774280b9 | ||
|
|
74bd2557af | ||
|
|
605fc8ecd8 | ||
|
|
6b745c53dc | ||
|
|
98c60b38fc | ||
|
|
be4515ec9a | ||
|
|
dcb1b64696 | ||
|
|
1190a60c7a | ||
|
|
82ecf207c6 | ||
|
|
84c7f1bf58 | ||
|
|
5ac453fe07 | ||
|
|
c2f39d653d | ||
|
|
748178f536 | ||
|
|
ffbef69dc7 | ||
|
|
3757124323 | ||
|
|
e65f214b3c | ||
|
|
f14004e56b | ||
|
|
dadc004dd6 | ||
|
|
3697d9e1f0 | ||
|
|
94524f7330 | ||
|
|
17d8a3e5d6 | ||
|
|
39996d7612 | ||
|
|
601ee4320f | ||
|
|
6e4f6fa92d | ||
|
|
b7103c29dd | ||
|
|
e53209e5ae | ||
|
|
501fbd3114 | ||
|
|
5626f43e2b | ||
|
|
0185cb48fb | ||
|
|
9a49eb06da | ||
|
|
3fbc012231 | ||
|
|
3938cf33d8 | ||
|
|
38b3d0c87c | ||
|
|
15a2fa6199 | ||
|
|
e834209d40 | ||
|
|
9607d21828 | ||
|
|
6fbbf36143 | ||
|
|
db25f3282e | ||
|
|
ceb92838e7 | ||
|
|
afb98f2cfc | ||
|
|
ed785fb087 | ||
|
|
3921180d77 | ||
|
|
ef36e0311e | ||
|
|
bf4cbe1204 | ||
|
|
28eee4da25 | ||
|
|
933ccf6deb | ||
|
|
9ac41322e5 |
No files matched your search
@@ -0,0 +1,79 @@
|
||||
name: ci
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- working-directory: lbry
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
- datanetwork
|
||||
- blockchain
|
||||
- other
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: sudo apt install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(matrix.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
+5
-4
@@ -46,14 +46,14 @@ test:datanetwork-integration:
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
@@ -84,14 +84,15 @@ test:json-api:
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.58.0"
|
||||
__version__ = "0.63.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+24
-13
@@ -467,12 +467,17 @@ class TranscodeConfig(BaseConfig):
|
||||
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 18 -vf "format=yuv420p"')
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 192k')
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
class CLIConfig(TranscodeConfig):
|
||||
@@ -558,6 +563,7 @@ class Config(CLIConfig):
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
@@ -573,15 +579,15 @@ class Config(CLIConfig):
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv1.lbry.com', 50001),
|
||||
('spv2.lbry.com', 50001),
|
||||
('spv3.lbry.com', 50001),
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
('spv13.lbry.com', 50001),
|
||||
('spv14.lbry.com', 50001),
|
||||
('spv15.lbry.com', 50001),
|
||||
('spv16.lbry.com', 50001),
|
||||
('spv17.lbry.com', 50001),
|
||||
('spv18.lbry.com', 50001),
|
||||
('spv19.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
@@ -601,7 +607,7 @@ class Config(CLIConfig):
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
@@ -616,6 +622,11 @@ class Config(CLIConfig):
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
STRATEGIES, "standard")
|
||||
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
)
|
||||
|
||||
@property
|
||||
def streaming_host(self):
|
||||
return self.streaming_server.split(':')[0]
|
||||
|
||||
@@ -53,10 +53,13 @@ Code | Name | Message
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
+56
-3
@@ -1,4 +1,4 @@
|
||||
from .base import BaseError
|
||||
from .base import BaseError, claim_id
|
||||
|
||||
|
||||
class UserInputError(BaseError):
|
||||
@@ -16,18 +16,22 @@ class CommandError(UserInputError):
|
||||
class CommandDoesNotExistError(CommandError):
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' does not exist.")
|
||||
|
||||
|
||||
class CommandDeprecatedError(CommandError):
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is deprecated.")
|
||||
|
||||
|
||||
class CommandInvalidArgumentError(CommandError):
|
||||
|
||||
def __init__(self, argument, command):
|
||||
self.argument = argument
|
||||
self.command = command
|
||||
super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
|
||||
|
||||
|
||||
@@ -37,6 +41,7 @@ class CommandTemporarilyUnavailableError(CommandError):
|
||||
"""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is temporarily unavailable.")
|
||||
|
||||
|
||||
@@ -46,6 +51,7 @@ class CommandPermanentlyUnavailableError(CommandError):
|
||||
"""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is permanently unavailable.")
|
||||
|
||||
|
||||
@@ -58,12 +64,15 @@ class InputValueError(UserInputError, ValueError):
|
||||
class GenericInputValueError(InputValueError):
|
||||
|
||||
def __init__(self, value, argument):
|
||||
self.value = value
|
||||
self.argument = argument
|
||||
super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
|
||||
|
||||
|
||||
class InputValueIsNoneError(InputValueError):
|
||||
|
||||
def __init__(self, argument):
|
||||
self.argument = argument
|
||||
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
||||
|
||||
|
||||
@@ -79,6 +88,7 @@ class ConfigWriteError(ConfigurationError):
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Cannot write configuration file '{path}'.")
|
||||
|
||||
|
||||
@@ -88,6 +98,7 @@ class ConfigReadError(ConfigurationError):
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Cannot find provided configuration file '{path}'.")
|
||||
|
||||
|
||||
@@ -97,18 +108,21 @@ class ConfigParseError(ConfigurationError):
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Failed to parse the configuration file '{path}'.")
|
||||
|
||||
|
||||
class ConfigMissingError(ConfigurationError):
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
|
||||
|
||||
|
||||
class ConfigInvalidError(ConfigurationError):
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Configuration file '{path}' has setting with invalid value.")
|
||||
|
||||
|
||||
@@ -188,24 +202,29 @@ class DataDownloadError(WalletError):
|
||||
class ResolveError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Failed to resolve '{url}'.")
|
||||
|
||||
|
||||
class ResolveTimeoutError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Failed to resolve '{url}' within the timeout.")
|
||||
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_id):
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
|
||||
def __init__(self, url, censor_hash):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
@@ -218,9 +237,32 @@ class InvalidPasswordError(WalletError):
|
||||
class IncompatibleWalletServerError(WalletError):
|
||||
|
||||
def __init__(self, server, port):
|
||||
self.server = server
|
||||
self.port = port
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
self.address = address
|
||||
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentWalletLockedError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, daily_fee, max_fee):
|
||||
self.daily_fee = daily_fee
|
||||
self.max_fee = max_fee
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
@@ -278,30 +320,35 @@ class DownloadCancelledError(BlobError):
|
||||
class DownloadSDTimeoutError(BlobError):
|
||||
|
||||
def __init__(self, download):
|
||||
self.download = download
|
||||
super().__init__(f"Failed to download sd blob {download} within timeout.")
|
||||
|
||||
|
||||
class DownloadDataTimeoutError(BlobError):
|
||||
|
||||
def __init__(self, download):
|
||||
self.download = download
|
||||
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidDataError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidBlobHashError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
@@ -314,12 +361,14 @@ class ComponentError(BaseError):
|
||||
class ComponentStartConditionNotMetError(ComponentError):
|
||||
|
||||
def __init__(self, components):
|
||||
self.components = components
|
||||
super().__init__(f"Unresolved dependencies for: {components}")
|
||||
|
||||
|
||||
class ComponentsNotStartedError(ComponentError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
@@ -332,16 +381,20 @@ class CurrencyExchangeError(BaseError):
|
||||
class InvalidExchangeRateResponseError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, source, reason):
|
||||
self.source = source
|
||||
self.reason = reason
|
||||
super().__init__(f"Failed to get exchange rate from {source}: {reason}")
|
||||
|
||||
|
||||
class CurrencyConversionError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidCurrencyError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, currency):
|
||||
self.currency = currency
|
||||
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
|
||||
@@ -1,2 +1,9 @@
|
||||
from binascii import hexlify
|
||||
|
||||
|
||||
def claim_id(claim_hash):
|
||||
return hexlify(claim_hash[::-1]).decode()
|
||||
|
||||
|
||||
class BaseError(Exception):
|
||||
pass
|
||||
+17
-4
@@ -13,10 +13,12 @@ class {name}({parents}):{doc}
|
||||
"""
|
||||
|
||||
INIT = """
|
||||
def __init__({args}):
|
||||
def __init__({args}):{fields}
|
||||
super().__init__({format}"{message}")
|
||||
"""
|
||||
|
||||
FUNCTIONS = ['claim_id']
|
||||
|
||||
|
||||
class ErrorClass:
|
||||
|
||||
@@ -50,10 +52,20 @@ class ErrorClass:
|
||||
|
||||
def get_arguments(self):
|
||||
args = ['self']
|
||||
for arg in re.findall('{([a-z0-1_]+)}', self.message):
|
||||
for arg in re.findall('{([a-z0-1_()]+)}', self.message):
|
||||
for func in FUNCTIONS:
|
||||
if arg.startswith(f'{func}('):
|
||||
arg = arg[len(f'{func}('):-1]
|
||||
break
|
||||
args.append(arg)
|
||||
return args
|
||||
|
||||
@staticmethod
|
||||
def get_fields(args):
|
||||
if len(args) > 1:
|
||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def get_doc_string(doc):
|
||||
if doc:
|
||||
@@ -69,7 +81,8 @@ class ErrorClass:
|
||||
args = self.get_arguments()
|
||||
if self.is_leaf:
|
||||
out.write((CLASS + INIT).format(
|
||||
name=self.class_name, parents=', '.join(parents), args=', '.join(args),
|
||||
name=self.class_name, parents=', '.join(parents),
|
||||
args=', '.join(args), fields=self.get_fields(args),
|
||||
message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
|
||||
))
|
||||
else:
|
||||
@@ -102,7 +115,7 @@ def find_parent(stack, child):
|
||||
|
||||
|
||||
def generate(out):
|
||||
out.write('from .base import BaseError\n')
|
||||
out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
|
||||
stack = {}
|
||||
for error in get_errors():
|
||||
error.render(out, find_parent(stack, error))
|
||||
|
||||
+3
-8
@@ -3,7 +3,6 @@ import sys
|
||||
import shutil
|
||||
import signal
|
||||
import pathlib
|
||||
import platform
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -256,16 +255,12 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
loggly_handler = get_loggly_handler(conf)
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
if sys.version_info < (3, 8) and platform.system() == "Windows":
|
||||
# TODO: remove after we move to requiring Python 3.8
|
||||
asyncio.set_event_loop(asyncio.ProactorEventLoop())
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
|
||||
@@ -110,7 +110,6 @@ class AnalyticsManager:
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
@@ -118,20 +117,24 @@ class AnalyticsManager:
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self.conf.share_usage_data
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
async def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
if self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
if self.enabled:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
@@ -154,7 +157,7 @@ class AnalyticsManager:
|
||||
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
if self.enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post(event)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,6 +30,7 @@ log = logging.getLogger(__name__)
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
@@ -119,6 +121,7 @@ class WalletComponent(Component):
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
@@ -160,6 +163,34 @@ class WalletComponent(Component):
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class WalletServerPaymentsComponent(Component):
|
||||
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
depends_on = [WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.usage_payment_service = WalletServerPayer(
|
||||
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
|
||||
)
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[WalletServerPayer]:
|
||||
return self.usage_payment_service
|
||||
|
||||
async def start(self):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
|
||||
|
||||
async def stop(self):
|
||||
await self.usage_payment_service.stop()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'max_fee': self.usage_payment_service.max_fee,
|
||||
'running': self.usage_payment_service.running
|
||||
}
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
@@ -297,6 +297,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
|
||||
self.conf = conf
|
||||
self.platform_info = system_info.get_platform()
|
||||
self._video_file_analyzer = VideoFileAnalyzer(conf)
|
||||
self._node_id = None
|
||||
self._installation_id = None
|
||||
@@ -307,7 +308,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
skip_components=conf.components_to_skip or []
|
||||
)
|
||||
self.component_startup_task = None
|
||||
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
|
||||
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
|
||||
|
||||
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
|
||||
rpc_app = web.Application()
|
||||
@@ -357,7 +358,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@classmethod
|
||||
def get_api_definitions(cls):
|
||||
prefix = 'jsonrpc_'
|
||||
not_grouped = ['routing_table_get']
|
||||
not_grouped = ['routing_table_get', 'ffmpeg_find']
|
||||
api = {
|
||||
'groups': {
|
||||
group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip()
|
||||
@@ -449,7 +450,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def start(self):
|
||||
log.info("Starting LBRYNet Daemon")
|
||||
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
|
||||
log.info("Platform: %s", json.dumps(system_info.get_platform(), indent=2))
|
||||
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
|
||||
await self.analytics_manager.send_server_startup()
|
||||
await self.rpc_runner.setup()
|
||||
await self.streaming_runner.setup()
|
||||
@@ -764,6 +765,26 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
asyncio.get_event_loop().call_later(0, shutdown)
|
||||
return "Shutting down"
|
||||
|
||||
async def jsonrpc_ffmpeg_find(self):
|
||||
"""
|
||||
Get ffmpeg installation information
|
||||
|
||||
Usage:
|
||||
ffmpeg_find
|
||||
|
||||
Options:
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary of ffmpeg information
|
||||
{
|
||||
'available': (bool) found ffmpeg,
|
||||
'which': (str) path to ffmpeg,
|
||||
'analyze_audio_volume': (bool) should ffmpeg analyze audio
|
||||
}
|
||||
"""
|
||||
return await self._video_file_analyzer.status(recheck=True)
|
||||
|
||||
async def jsonrpc_status(self):
|
||||
"""
|
||||
Get daemon status
|
||||
@@ -894,9 +915,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'build': (str) "dev" | "qa" | "rc" | "release",
|
||||
}
|
||||
"""
|
||||
platform_info = system_info.get_platform()
|
||||
log.info("Get version info: %s", json.dumps(platform_info))
|
||||
return platform_info
|
||||
return self.platform_info
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None):
|
||||
@@ -2154,7 +2173,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
List my stream and channel claims.
|
||||
|
||||
Usage:
|
||||
claim_list [--claim_type=<claim_type>]
|
||||
claim_list [--claim_type=<claim_type>...]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
[--resolve]
|
||||
@@ -3123,8 +3142,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
|
||||
file_path = await self._video_file_analyzer.verify_or_repair(validate_file, optimize_file, file_path)
|
||||
|
||||
file_path = await self._video_file_analyzer.verify_or_repair(
|
||||
validate_file, optimize_file, file_path, ignore_non_video=True
|
||||
)
|
||||
claim = Claim()
|
||||
claim.stream.update(file_path=file_path, sd_hash='0' * 96, **kwargs)
|
||||
tx = await Transaction.claim_create(
|
||||
@@ -4565,7 +4585,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Usage:
|
||||
comment_create (<comment> | --comment=<comment>)
|
||||
(<claim_id> | --claim_id=<claim_id> | --parent_id=<parent_id>)
|
||||
[--channel_id=<channel_id>] [--channel_name=<channel_name>]
|
||||
(--channel_id=<channel_id> | --channel_name=<channel_name>)
|
||||
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
|
||||
|
||||
Options:
|
||||
@@ -4593,23 +4613,23 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
channel = await self.get_channel_or_error(
|
||||
wallet, channel_account_id, channel_id, channel_name, for_signing=True
|
||||
)
|
||||
|
||||
comment_body = {
|
||||
'comment': comment.strip(),
|
||||
'claim_id': claim_id,
|
||||
'parent_id': parent_id,
|
||||
'channel_id': channel.claim_id,
|
||||
'channel_name': channel.claim_name,
|
||||
}
|
||||
channel = await self.get_channel_or_none(
|
||||
wallet, channel_account_id, channel_id, channel_name, for_signing=True
|
||||
)
|
||||
if channel:
|
||||
comment_body.update({
|
||||
'channel_id': channel.claim_id,
|
||||
'channel_name': channel.claim_name,
|
||||
})
|
||||
comment_client.sign_comment(comment_body, channel)
|
||||
comment_client.sign_comment(comment_body, channel)
|
||||
|
||||
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
|
||||
if 'signature' in response:
|
||||
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
|
||||
response.update({
|
||||
'is_claim_signature_valid': comment_client.is_comment_signed_by_channel(response, channel)
|
||||
})
|
||||
return response
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -4853,7 +4873,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
async def resolve(self, accounts, urls):
|
||||
results = await self.ledger.resolve(accounts, urls)
|
||||
if results:
|
||||
if self.conf.save_resolved_claims and results:
|
||||
try:
|
||||
claims = self.stream_manager._convert_to_old_resolve_output(self.wallet_manager, results)
|
||||
await self.storage.save_claims_for_resolve([
|
||||
|
||||
@@ -294,7 +294,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'added_on': managed_stream.added_on,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
|
||||
@@ -3,10 +3,12 @@ import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
|
||||
import typing
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
@@ -36,17 +38,19 @@ class JsonFormatter(logging.Formatter):
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
def __init__(self, loggly_token: str, config: 'Config'):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.cookies = {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._config = config
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._config.share_usage_data
|
||||
|
||||
@staticmethod
|
||||
def get_full_message(record):
|
||||
@@ -62,12 +66,14 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry:
|
||||
if self._loop.is_running() and retry and self.enabled:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
|
||||
def emit(self, record):
|
||||
if not self.enabled:
|
||||
return
|
||||
try:
|
||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
||||
except RuntimeError: # TODO: use a second loop
|
||||
@@ -83,7 +89,7 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
pass
|
||||
|
||||
|
||||
def get_loggly_handler():
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
|
||||
def get_loggly_handler(config):
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
|
||||
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee):
|
||||
raw_content_fee, fully_reflected):
|
||||
return {
|
||||
"rowid": rowid,
|
||||
"added_on": added_on,
|
||||
@@ -129,7 +129,8 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
),
|
||||
"fully_reflected": fully_reflected
|
||||
}
|
||||
|
||||
|
||||
@@ -138,11 +139,14 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
|
||||
"select file.rowid, file.*, stream.*, c.* "
|
||||
"select file.rowid, file.*, stream.*, c.*, "
|
||||
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
|
||||
" is null then 0 else 1 end as fully_reflected "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"order by c.rowid desc").fetchall():
|
||||
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
|
||||
claim = StoredContentClaim(*claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -152,7 +156,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
_get_lbry_file_stream_dict(
|
||||
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee
|
||||
raw_content_fee, fully_reflected
|
||||
)
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
|
||||
+72
-47
@@ -6,35 +6,53 @@ import pathlib
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import platform
|
||||
|
||||
import lbry.utils
|
||||
from lbry.conf import TranscodeConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
DISABLED = platform.system() == "Windows"
|
||||
|
||||
|
||||
class VideoFileAnalyzer:
|
||||
|
||||
def _replace_or_pop_env(self, variable):
|
||||
if variable + '_ORIG' in self._env_copy:
|
||||
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
|
||||
else:
|
||||
self._env_copy.pop(variable, None)
|
||||
|
||||
def __init__(self, conf: TranscodeConfig):
|
||||
self._conf = conf
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
self._checked_ffmpeg = False
|
||||
self._env_copy = dict(os.environ)
|
||||
if lbry.utils.is_running_from_bundle():
|
||||
# handle the situation where PyInstaller overrides our runtime environment:
|
||||
self._replace_or_pop_env('LD_LIBRARY_PATH')
|
||||
|
||||
async def _execute(self, command, arguments):
|
||||
if DISABLED:
|
||||
return "Disabled on Windows", -1
|
||||
args = shlex.split(arguments)
|
||||
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
os.path.join(self._conf.ffmpeg_folder, command), *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env_copy
|
||||
)
|
||||
stdout, stderr = await process.communicate() # returns when the streams are closed
|
||||
return stdout.decode() + stderr.decode(), process.returncode
|
||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||
|
||||
async def _verify_executable(self, name):
|
||||
try:
|
||||
version, code = await self._execute(name, "-version")
|
||||
except Exception as e:
|
||||
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
|
||||
code = -1
|
||||
version = ""
|
||||
version = str(e)
|
||||
if code != 0 or not version.startswith(name):
|
||||
log.warning("Unable to run %s, but it was requested. Code: %d; Message: %s", name, code, version)
|
||||
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
|
||||
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
|
||||
return version
|
||||
@@ -44,22 +62,24 @@ class VideoFileAnalyzer:
|
||||
return
|
||||
await self._verify_executable("ffprobe")
|
||||
version = await self._verify_executable("ffmpeg")
|
||||
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
|
||||
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
|
||||
self._ffmpeg_installed = True
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
||||
|
||||
async def status(self, reset=False):
|
||||
async def status(self, reset=False, recheck=False):
|
||||
if reset:
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
|
||||
installed = True
|
||||
try:
|
||||
await self._verify_ffmpeg_installed()
|
||||
except FileNotFoundError:
|
||||
installed = False
|
||||
|
||||
if self._checked_ffmpeg and not recheck:
|
||||
installed = self._ffmpeg_installed
|
||||
else:
|
||||
installed = True
|
||||
try:
|
||||
await self._verify_ffmpeg_installed()
|
||||
except FileNotFoundError:
|
||||
installed = False
|
||||
self._checked_ffmpeg = True
|
||||
return {
|
||||
"available": installed,
|
||||
"which": self._which,
|
||||
@@ -92,24 +112,21 @@ class VideoFileAnalyzer:
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _verify_bitrate(scan_data: json):
|
||||
if "bit_rate" not in scan_data["format"]:
|
||||
def _verify_bitrate(self, scan_data: json, file_path):
|
||||
bit_rate_max = float(self._conf.video_bitrate_maximum)
|
||||
if bit_rate_max <= 0:
|
||||
return ""
|
||||
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
|
||||
pixels = -1.0
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
pieces = stream["r_frame_rate"].split('/', 1)
|
||||
frame_rate = float(pieces[0]) if len(pieces) == 1 \
|
||||
else float(pieces[0]) / float(pieces[1])
|
||||
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
|
||||
if "bit_rate" in scan_data["format"]:
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
else:
|
||||
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
|
||||
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
|
||||
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
|
||||
|
||||
if pixels > 0.0 and pixels / bit_rate < 3.0:
|
||||
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
|
||||
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
|
||||
if bit_rate > bit_rate_max:
|
||||
return "The bit rate is above the configured maximum. Actual: " \
|
||||
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -119,10 +136,9 @@ class VideoFileAnalyzer:
|
||||
return ""
|
||||
|
||||
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
|
||||
iterator = re.finditer(r"\s+seeks:(\d+)\s+", result)
|
||||
for match in iterator:
|
||||
if int(match.group(1)) != 0:
|
||||
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
|
||||
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
|
||||
if match and int(match.group(1)) != 0:
|
||||
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
@@ -174,6 +190,9 @@ class VideoFileAnalyzer:
|
||||
# https://developers.google.com/media/vp9/settings/vod/
|
||||
return int(-0.011 * height + 40)
|
||||
|
||||
def _get_video_scaler(self):
|
||||
return self._conf.video_scaler
|
||||
|
||||
async def _get_video_encoder(self, scan_data):
|
||||
# use what the user said if it's there:
|
||||
# if it's not there, use h264 if we can because it's way faster than the others
|
||||
@@ -253,12 +272,12 @@ class VideoFileAnalyzer:
|
||||
continue
|
||||
codec = stream["codec_name"].split(",")
|
||||
if "theora" in codec:
|
||||
return "ogg"
|
||||
return "ogv"
|
||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||
return "webm"
|
||||
|
||||
if "theora" in video_encoder:
|
||||
return "ogg"
|
||||
return "ogv"
|
||||
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||
return "webm"
|
||||
return "mp4"
|
||||
@@ -270,31 +289,35 @@ class VideoFileAnalyzer:
|
||||
scan_data = json.loads(result)
|
||||
except Exception as e:
|
||||
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
|
||||
if validate:
|
||||
raise Exception(f'Invalid video file: {file_path}')
|
||||
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
|
||||
return
|
||||
raise ValueError(f'Absent or unreadable video file: {file_path}')
|
||||
|
||||
if "format" not in scan_data:
|
||||
if validate:
|
||||
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
|
||||
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
|
||||
return
|
||||
if "format" not in scan_data or "duration" not in scan_data["format"]:
|
||||
log.debug("Format data is missing from ffprobe results for: %s", file_path)
|
||||
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
|
||||
|
||||
if float(scan_data["format"]["duration"]) < 0.1:
|
||||
log.debug("Media file appears to be an image: %s", file_path)
|
||||
raise ValueError(f'Assuming image file at: {file_path}')
|
||||
|
||||
return scan_data
|
||||
|
||||
async def verify_or_repair(self, validate, repair, file_path):
|
||||
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
if not validate and not repair:
|
||||
return file_path
|
||||
|
||||
await self._verify_ffmpeg_installed()
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
try:
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
except ValueError:
|
||||
if ignore_non_video:
|
||||
return file_path
|
||||
raise
|
||||
|
||||
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
||||
log.debug("Analyzing %s:", file_path)
|
||||
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
||||
container_msg = self._verify_container(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data, file_path)
|
||||
video_msg = self._verify_video_encoding(scan_data)
|
||||
audio_msg = self._verify_audio_encoding(scan_data)
|
||||
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
|
||||
@@ -320,6 +343,8 @@ class VideoFileAnalyzer:
|
||||
if video_msg or bitrate_msg:
|
||||
video_encoder = await self._get_video_encoder(scan_data)
|
||||
transcode_command.append(video_encoder)
|
||||
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
|
||||
transcode_command.append(self._get_video_scaler())
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
|
||||
@@ -148,6 +148,7 @@ types_map = {
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.oga': ('audio/ogg', 'audio'),
|
||||
'.ogv': ('video/ogg', 'video'),
|
||||
'.ogg': ('video/ogg', 'video'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
|
||||
+39
-32
@@ -13,6 +13,16 @@ NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
|
||||
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
|
||||
|
||||
|
||||
def set_reference(reference, claim_hash, rows):
|
||||
if claim_hash:
|
||||
for txo in rows:
|
||||
if claim_hash == txo['claim_hash']:
|
||||
reference.tx_hash = txo['txo_hash'][:32]
|
||||
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
|
||||
reference.height = txo['height']
|
||||
return
|
||||
|
||||
|
||||
class Censor:
|
||||
|
||||
__slots__ = 'streams', 'channels', 'censored', 'total'
|
||||
@@ -28,7 +38,9 @@ class Censor:
|
||||
for claim_hash, lookup in (
|
||||
(row['claim_hash'], self.streams),
|
||||
(row['claim_hash'], self.channels),
|
||||
(row['channel_hash'], self.channels)):
|
||||
(row['channel_hash'], self.channels),
|
||||
(row['reposted_claim_hash'], self.streams),
|
||||
(row['reposted_claim_hash'], self.channels)):
|
||||
censoring_channel_hash = lookup.get(claim_hash)
|
||||
if censoring_channel_hash:
|
||||
was_censored = True
|
||||
@@ -39,12 +51,12 @@ class Censor:
|
||||
self.total += 1
|
||||
return was_censored
|
||||
|
||||
def to_message(self, outputs: OutputsMessage):
|
||||
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
|
||||
outputs.blocked_total = self.total
|
||||
for censoring_channel_hash, count in self.censored.items():
|
||||
block = outputs.blocked.add()
|
||||
block.count = count
|
||||
block.channel_hash = censoring_channel_hash
|
||||
blocked = outputs.blocked.add()
|
||||
blocked.count = count
|
||||
set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
|
||||
|
||||
|
||||
class Outputs:
|
||||
@@ -66,26 +78,35 @@ class Outputs:
|
||||
for txo_message in self.extra_txos:
|
||||
self.message_to_txo(txo_message, tx_map)
|
||||
txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
|
||||
return txos, self.inflate_blocked()
|
||||
return txos, self.inflate_blocked(tx_map)
|
||||
|
||||
def inflate_blocked(self):
|
||||
def inflate_blocked(self, tx_map):
|
||||
return {
|
||||
"total": self.blocked_total,
|
||||
"channels": {
|
||||
hexlify(message.channel_hash[::-1]).decode(): message.count
|
||||
for message in self.blocked
|
||||
}
|
||||
"channels": [{
|
||||
'channel': self.message_to_txo(blocked.channel, tx_map),
|
||||
'blocked': blocked.count
|
||||
} for blocked in self.blocked]
|
||||
}
|
||||
|
||||
def message_to_txo(self, txo_message, tx_map):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
return {
|
||||
error = {
|
||||
'error': {
|
||||
'name': txo_message.error.Code.Name(txo_message.error.code).lower(),
|
||||
'name': txo_message.error.Code.Name(txo_message.error.code),
|
||||
'text': txo_message.error.text,
|
||||
}
|
||||
}
|
||||
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
|
||||
if error['error']['name'] == BLOCKED:
|
||||
error['error']['censor'] = self.message_to_txo(
|
||||
txo_message.error.blocked.channel, tx_map
|
||||
)
|
||||
return error
|
||||
|
||||
tx = tx_map.get(txo_message.tx_hash)
|
||||
if not tx:
|
||||
return
|
||||
txo = tx.outputs[txo_message.nout]
|
||||
if txo_message.WhichOneof('meta') == 'claim':
|
||||
claim = txo_message.claim
|
||||
txo.meta = {
|
||||
@@ -145,7 +166,7 @@ class Outputs:
|
||||
if total is not None:
|
||||
page.total = total
|
||||
if blocked is not None:
|
||||
blocked.to_message(page)
|
||||
blocked.to_message(page, extra_txo_rows)
|
||||
for row in txo_rows:
|
||||
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
|
||||
for row in extra_txo_rows:
|
||||
@@ -162,6 +183,7 @@ class Outputs:
|
||||
txo_message.error.code = ErrorMessage.NOT_FOUND
|
||||
elif isinstance(txo, ResolveCensoredError):
|
||||
txo_message.error.code = ErrorMessage.BLOCKED
|
||||
set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
|
||||
return
|
||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
@@ -184,20 +206,5 @@ class Outputs:
|
||||
txo_message.claim.trending_mixed = txo['trending_mixed']
|
||||
txo_message.claim.trending_local = txo['trending_local']
|
||||
txo_message.claim.trending_global = txo['trending_global']
|
||||
cls.set_reference(txo_message, 'channel', txo['channel_hash'], extra_txo_rows)
|
||||
cls.set_reference(txo_message, 'repost', txo['reposted_claim_hash'], extra_txo_rows)
|
||||
|
||||
@staticmethod
|
||||
def set_blocked(message, blocked):
|
||||
message.blocked_total = blocked.total
|
||||
|
||||
@staticmethod
|
||||
def set_reference(message, attr, claim_hash, rows):
|
||||
if claim_hash:
|
||||
for txo in rows:
|
||||
if claim_hash == txo['claim_hash']:
|
||||
reference = getattr(message.claim, attr)
|
||||
reference.tx_hash = txo['txo_hash'][:32]
|
||||
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
|
||||
reference.height = txo['height']
|
||||
break
|
||||
set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
|
||||
set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
|
||||
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='result.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\".\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x14\n\x0c\x63hannel_hash\x18\x02 \x01(\x0c\x62\x06proto3')
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.Outputb\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
@@ -388,9 +388,9 @@ _BLOCKED = _descriptor.Descriptor(
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel_hash', full_name='pb.Blocked.channel_hash', index=1,
|
||||
number=2, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
name='channel', full_name='pb.Blocked.channel', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
@@ -407,7 +407,7 @@ _BLOCKED = _descriptor.Descriptor(
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=884,
|
||||
serialized_end=930,
|
||||
serialized_end=937,
|
||||
)
|
||||
|
||||
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
|
||||
@@ -426,6 +426,7 @@ _CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
|
||||
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
|
||||
_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
|
||||
_ERROR_CODE.containing_type = _ERROR
|
||||
_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
|
||||
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
|
||||
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
|
||||
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ def _create_url_regex():
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
|
||||
@@ -111,6 +111,10 @@ class ManagedStream:
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def is_fully_reflected(self) -> bool:
|
||||
return self.fully_reflected.is_set()
|
||||
|
||||
@property
|
||||
def descriptor(self) -> StreamDescriptor:
|
||||
return self.downloader.descriptor
|
||||
@@ -429,9 +433,9 @@ class ManagedStream:
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
await protocol.send_handshake()
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd:
|
||||
if sent_sd: # reflector needed the sd blob
|
||||
sent.append(self.sd_hash)
|
||||
if not sent_sd and not needed:
|
||||
if not sent_sd and not needed: # reflector already has the stream
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
|
||||
@@ -116,7 +116,7 @@ class StreamManager:
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
added_on: Optional[int], fully_reflected: bool):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
@@ -127,6 +127,8 @@ class StreamManager:
|
||||
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
|
||||
analytics_manager=self.analytics_manager, added_on=added_on
|
||||
)
|
||||
if fully_reflected:
|
||||
stream.fully_reflected.set()
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
@@ -159,7 +161,7 @@ class StreamManager:
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee'],
|
||||
file_info['added_on']
|
||||
file_info['added_on'], file_info['fully_reflected']
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
|
||||
+7
-2
@@ -253,6 +253,11 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
lambda e: e.tx.id == txid
|
||||
)
|
||||
|
||||
def on_address_update(self, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.address == address
|
||||
)
|
||||
|
||||
def on_transaction_address(self, tx, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.tx.id == tx.id and e.address == address
|
||||
@@ -544,7 +549,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
)
|
||||
|
||||
async def resolve(self, uri):
|
||||
return await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
|
||||
|
||||
async def claim_search(self, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
|
||||
@@ -566,4 +571,4 @@ class CommandTestCase(IntegrationTestCase):
|
||||
return tx['outputs'][0]['claim_id']
|
||||
|
||||
def assertItemCount(self, result, count): # pylint: disable=invalid-name
|
||||
self.assertEqual(result['total_items'], count)
|
||||
self.assertEqual(count, result['total_items'])
|
||||
@@ -4,6 +4,7 @@ import datetime
|
||||
import random
|
||||
import socket
|
||||
import string
|
||||
import sys
|
||||
import json
|
||||
import typing
|
||||
import asyncio
|
||||
@@ -276,3 +277,8 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled
|
||||
return response['data']['ip']
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.3/lbrycrd-linux-1743.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
+16
-10
@@ -134,16 +134,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
col, op = col[:-len('__not_like')], 'NOT LIKE'
|
||||
elif key.endswith('__in') or key.endswith('__not_in'):
|
||||
if key.endswith('__in'):
|
||||
col, op = col[:-len('__in')], 'IN'
|
||||
col, op, one_val_op = col[:-len('__in')], 'IN', '='
|
||||
else:
|
||||
col, op = col[:-len('__not_in')], 'NOT IN'
|
||||
col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!='
|
||||
if constraint:
|
||||
if isinstance(constraint, (list, set, tuple)):
|
||||
keys = []
|
||||
for i, val in enumerate(constraint):
|
||||
keys.append(f':{key}{tag}_{i}')
|
||||
values[f'{key}{tag}_{i}'] = val
|
||||
sql.append(f'{col} {op} ({", ".join(keys)})')
|
||||
if len(constraint) == 1:
|
||||
values[f'{key}{tag}'] = next(iter(constraint))
|
||||
sql.append(f'{col} {one_val_op} :{key}{tag}')
|
||||
else:
|
||||
keys = []
|
||||
for i, val in enumerate(constraint):
|
||||
keys.append(f':{key}{tag}_{i}')
|
||||
values[f'{key}{tag}_{i}'] = val
|
||||
sql.append(f'{col} {op} ({", ".join(keys)})')
|
||||
elif isinstance(constraint, str):
|
||||
sql.append(f'{col} {op} ({constraint})')
|
||||
else:
|
||||
@@ -739,9 +743,11 @@ class Database(SQLiteMixin):
|
||||
|
||||
@staticmethod
|
||||
def constrain_claims(constraints):
|
||||
claim_type = constraints.pop('claim_type', None)
|
||||
if claim_type is not None:
|
||||
constraints['txo_type'] = TXO_TYPES[claim_type]
|
||||
claim_types = constraints.pop('claim_type', None)
|
||||
if isinstance(claim_types, str) and claim_types:
|
||||
claim_types = [claim_types]
|
||||
if isinstance(claim_types, list) and claim_types:
|
||||
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
|
||||
else:
|
||||
constraints['txo_type__in'] = CLAIM_TYPES
|
||||
|
||||
|
||||
@@ -630,7 +630,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||
|
||||
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
|
||||
outputs = Outputs.from_base64(await query)
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs = []
|
||||
if len(outputs.txs) > 0:
|
||||
txs: List[Transaction] = await asyncio.gather(*(
|
||||
@@ -738,6 +739,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
for txo in txos:
|
||||
resolved = response[txo.permanent_url]
|
||||
if isinstance(resolved, Output):
|
||||
resolved.update_annotations(txo)
|
||||
results.append(resolved)
|
||||
else:
|
||||
if isinstance(resolved, dict) and 'error' in resolved:
|
||||
|
||||
+14
-2
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
from time import perf_counter
|
||||
from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
@@ -68,7 +69,14 @@ class ClientSession(BaseClientSession):
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
return request.result()
|
||||
try:
|
||||
return request.result()
|
||||
except ConnectionResetError:
|
||||
log.error(
|
||||
"wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes",
|
||||
self.server[0], method, len(args), len(json.dumps(args))
|
||||
)
|
||||
raise
|
||||
except (RPCError, ProtocolError) as e:
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
@@ -148,12 +156,13 @@ class ClientSession(BaseClientSession):
|
||||
class Network:
|
||||
|
||||
PROTOCOL_VERSION = __version__
|
||||
MINIMUM_REQUIRED = (0, 57, 0)
|
||||
MINIMUM_REQUIRED = (0, 59, 0)
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
@@ -181,17 +190,20 @@ class Network:
|
||||
while self.running:
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self.server_features = await self.get_server_features()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
|
||||
@@ -173,6 +173,9 @@ class SPVNode:
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
conf = {
|
||||
'DESCRIPTION': '',
|
||||
'PAYMENT_ADDRESS': '',
|
||||
'DAILY_FEE': '0',
|
||||
'DB_DIRECTORY': self.data_path,
|
||||
'DAEMON_URL': blockchain_node.rpc_url,
|
||||
'REORG_LIMIT': '100',
|
||||
@@ -228,6 +231,7 @@ class BlockchainProcess(asyncio.SubprocessProtocol):
|
||||
|
||||
def process_exited(self):
|
||||
self.stopped.set()
|
||||
self.ready.set()
|
||||
|
||||
|
||||
class BlockchainNode:
|
||||
@@ -315,6 +319,7 @@ class BlockchainNode:
|
||||
BlockchainProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
try:
|
||||
|
||||
@@ -380,14 +380,17 @@ class RPCSession(SessionBase):
|
||||
def __init__(self, *, framer=None, loop=None, connection=None):
|
||||
super().__init__(framer=framer, loop=loop)
|
||||
self.connection = connection or self.default_connection()
|
||||
self.client_version = 'unknown'
|
||||
|
||||
async def _receive_messages(self):
|
||||
while not self.is_closing():
|
||||
try:
|
||||
message = await self.framer.receive_message()
|
||||
except MemoryError as e:
|
||||
self.logger.warning(f'{e!r}')
|
||||
continue
|
||||
except MemoryError:
|
||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||
self._address[0], self._address[1])
|
||||
self._close()
|
||||
return
|
||||
|
||||
self.last_recv = time.perf_counter()
|
||||
self.recv_count += 1
|
||||
@@ -419,12 +422,18 @@ class RPCSession(SessionBase):
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
message = request.send_result(result)
|
||||
RESPONSE_TIMES.labels(method=request.method).observe(time.perf_counter() - start)
|
||||
RESPONSE_TIMES.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).observe(time.perf_counter() - start)
|
||||
if message:
|
||||
await self._send_message(message)
|
||||
if isinstance(result, Exception):
|
||||
self._bump_errors()
|
||||
REQUEST_ERRORS_COUNT.labels(method=request.method).inc()
|
||||
REQUEST_ERRORS_COUNT.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).inc()
|
||||
|
||||
def connection_lost(self, exc):
|
||||
# Cancel pending requests and message processing
|
||||
@@ -458,8 +467,8 @@ class RPCSession(SessionBase):
|
||||
async def send_notification(self, method, args=()):
|
||||
"""Send an RPC notification over the network."""
|
||||
message = self.connection.send_notification(Notification(method, args))
|
||||
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
await self._send_message(message)
|
||||
NOTIFICATION_COUNT.labels(method=method).inc()
|
||||
|
||||
def send_batch(self, raise_errors=False):
|
||||
"""Return a BatchRequest. Intended to be used like so:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import time
|
||||
import asyncio
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
import lbry
|
||||
from lbry.schema.claim import Claim
|
||||
@@ -9,6 +10,7 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -144,6 +146,7 @@ class BlockProcessor:
|
||||
self.blocks_event = asyncio.Event()
|
||||
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
|
||||
# Meta
|
||||
self.next_cache_check = 0
|
||||
@@ -163,6 +166,8 @@ class BlockProcessor:
|
||||
# is consistent with self.height
|
||||
self.state_lock = asyncio.Lock()
|
||||
|
||||
self.search_cache = {}
|
||||
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
# cancellations from shutdown don't lose work - when the task
|
||||
@@ -171,7 +176,7 @@ class BlockProcessor:
|
||||
# consistent and not being updated elsewhere.
|
||||
async def run_in_thread_locked():
|
||||
async with self.state_lock:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, func, *args)
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, func, *args)
|
||||
return await asyncio.shield(run_in_thread_locked())
|
||||
|
||||
async def check_and_advance_blocks(self, raw_blocks):
|
||||
@@ -188,14 +193,17 @@ class BlockProcessor:
|
||||
chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]]
|
||||
|
||||
if hprevs == chain:
|
||||
start = time.time()
|
||||
start = time.perf_counter()
|
||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
BLOCK_COUNT.set(self.height)
|
||||
BLOCK_UPDATE_TIMES.observe(processed_time)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'
|
||||
.format(len(blocks), s,
|
||||
time.time() - start))
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
if self._caught_up_event.is_set():
|
||||
await self.notifications.on_block(self.touched, self.height)
|
||||
self.touched = set()
|
||||
@@ -652,6 +660,7 @@ class BlockProcessor:
|
||||
self.logger.info('flushing to DB for a clean shutdown...')
|
||||
await self.flush(True)
|
||||
self.db.close()
|
||||
self.executor.shutdown(wait=True)
|
||||
|
||||
def force_chain_reorg(self, count):
|
||||
"""Force a reorg of the given number of blocks.
|
||||
@@ -727,7 +736,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
|
||||
self.sql: SQLDB = self.db.sql
|
||||
self.timer = Timer('BlockProcessor')
|
||||
self.search_cache = {}
|
||||
|
||||
def advance_blocks(self, blocks):
|
||||
self.sql.begin()
|
||||
@@ -742,8 +750,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
|
||||
@@ -3,6 +3,7 @@ import itertools
|
||||
import json
|
||||
import time
|
||||
from functools import wraps
|
||||
from pylru import lrucache
|
||||
|
||||
import aiohttp
|
||||
|
||||
@@ -42,6 +43,8 @@ class Daemon:
|
||||
self._height = None
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
self._block_hash_cache = lrucache(1000000)
|
||||
self._block_cache = lrucache(100000)
|
||||
|
||||
async def close(self):
|
||||
if self.connector:
|
||||
@@ -219,12 +222,25 @@ class Daemon:
|
||||
|
||||
async def block_hex_hashes(self, first, count):
|
||||
"""Return the hex hashes of count block starting at height first."""
|
||||
if first + count < (self.cached_height() or 0) - 200:
|
||||
return await self._cached_block_hex_hashes(first, count)
|
||||
params_iterable = ((h, ) for h in range(first, first + count))
|
||||
return await self._send_vector('getblockhash', params_iterable)
|
||||
|
||||
async def _cached_block_hex_hashes(self, first, count):
|
||||
"""Return the hex hashes of count block starting at height first."""
|
||||
cached = self._block_hash_cache.get((first, count))
|
||||
if cached:
|
||||
return cached
|
||||
params_iterable = ((h, ) for h in range(first, first + count))
|
||||
self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
|
||||
return self._block_hash_cache[(first, count)]
|
||||
|
||||
async def deserialised_block(self, hex_hash):
|
||||
"""Return the deserialised block with the given hex hash."""
|
||||
return await self._send_single('getblock', (hex_hash, True))
|
||||
if not self._block_cache.get(hex_hash):
|
||||
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
|
||||
return self._block_cache[hex_hash]
|
||||
|
||||
async def raw_blocks(self, hex_hashes):
|
||||
"""Return the raw binary blocks with the given hex hashes."""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
'repost': 3
|
||||
}
|
||||
|
||||
STREAM_TYPES = {
|
||||
|
||||
@@ -4,10 +4,11 @@ import apsw
|
||||
import logging
|
||||
from operator import itemgetter
|
||||
from typing import Tuple, List, Dict, Union, Type, Optional
|
||||
from binascii import unhexlify, hexlify
|
||||
from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
from itertools import chain
|
||||
from dataclasses import dataclass
|
||||
|
||||
from lbry.wallet.database import query, interpolate
|
||||
@@ -265,7 +266,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
else:
|
||||
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
|
||||
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
|
||||
@@ -281,15 +282,16 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'channel_ids' in constraints:
|
||||
channel_ids = constraints.pop('channel_ids')
|
||||
if channel_ids:
|
||||
constraints['claim.channel_hash__in'] = [
|
||||
constraints['claim.channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_ids
|
||||
]
|
||||
}
|
||||
if 'not_channel_ids' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_ids')
|
||||
if not_channel_ids:
|
||||
not_channel_ids_binary = [
|
||||
not_channel_ids_binary = {
|
||||
unhexlify(ncid)[::-1] for ncid in not_channel_ids
|
||||
]
|
||||
}
|
||||
constraints['claim.claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
|
||||
if constraints.get('has_channel_signature', False):
|
||||
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
|
||||
else:
|
||||
@@ -315,17 +317,23 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
constraints['claim.txo_hash'] = tx_hash + struct.pack('<I', nout)
|
||||
|
||||
if 'claim_type' in constraints:
|
||||
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
|
||||
claim_types = constraints.pop('claim_type')
|
||||
if isinstance(claim_types, str):
|
||||
claim_types = [claim_types]
|
||||
if claim_types:
|
||||
constraints['claim.claim_type__in'] = {
|
||||
CLAIM_TYPES[claim_type] for claim_type in claim_types
|
||||
}
|
||||
if 'stream_types' in constraints:
|
||||
stream_types = constraints.pop('stream_types')
|
||||
if stream_types:
|
||||
constraints['claim.stream_type__in'] = [
|
||||
constraints['claim.stream_type__in'] = {
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
]
|
||||
}
|
||||
if 'media_types' in constraints:
|
||||
media_types = constraints.pop('media_types')
|
||||
if media_types:
|
||||
constraints['claim.media_type__in'] = media_types
|
||||
constraints['claim.media_type__in'] = set(media_types)
|
||||
|
||||
if 'fee_currency' in constraints:
|
||||
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
|
||||
@@ -388,9 +396,13 @@ def search_claims(censor: Censor, **constraints) -> List:
|
||||
)
|
||||
|
||||
|
||||
def _get_referenced_rows(censor: Censor, txo_rows: List[dict]):
|
||||
def _get_referenced_rows(txo_rows: List[dict], censor_channels: List[bytes]):
|
||||
censor = ctx.get().get_resolve_censor()
|
||||
repost_hashes = set(filter(None, map(itemgetter('reposted_claim_hash'), txo_rows)))
|
||||
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
|
||||
channel_hashes = set(chain(
|
||||
filter(None, map(itemgetter('channel_hash'), txo_rows)),
|
||||
censor_channels
|
||||
))
|
||||
|
||||
reposted_txos = []
|
||||
if repost_hashes:
|
||||
@@ -413,12 +425,10 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
total = count_claims(**constraints)
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
context = ctx.get()
|
||||
search_censor = context.get_search_censor()
|
||||
txo_rows = search_claims(search_censor, **constraints)
|
||||
extra_txo_rows = _get_referenced_rows(context.get_resolve_censor(), txo_rows)
|
||||
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
|
||||
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
|
||||
|
||||
|
||||
@@ -426,7 +436,8 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
def resolve(urls) -> Tuple[List, List]:
|
||||
txo_rows = [resolve_url(raw_url) for raw_url in urls]
|
||||
extra_txo_rows = _get_referenced_rows(
|
||||
ctx.get().get_resolve_censor(), [r for r in txo_rows if isinstance(r, dict)]
|
||||
[txo for txo in txo_rows if isinstance(txo, dict)],
|
||||
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
|
||||
)
|
||||
return txo_rows, extra_txo_rows
|
||||
|
||||
@@ -452,7 +463,7 @@ def resolve_url(raw_url):
|
||||
if matches:
|
||||
channel = matches[0]
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, hexlify(next(iter(censor.censored))[::-1]).decode())
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
else:
|
||||
return LookupError(f'Could not find channel in "{raw_url}".')
|
||||
|
||||
@@ -472,13 +483,21 @@ def resolve_url(raw_url):
|
||||
if matches:
|
||||
return matches[0]
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, hexlify(next(iter(censor.censored))[::-1]).decode())
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
else:
|
||||
return LookupError(f'Could not find claim at "{raw_url}".')
|
||||
|
||||
return channel
|
||||
|
||||
|
||||
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
|
||||
CASE WHEN claim.claim_type = {CLAIM_TYPES['repost']}
|
||||
THEN claim.reposted_claim_hash
|
||||
ELSE claim.claim_hash
|
||||
END
|
||||
"""
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
@@ -498,7 +517,8 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
index_name = COMMON_TAGS[item]
|
||||
any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
|
||||
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
AND tag = '{item}'
|
||||
)
|
||||
"""
|
||||
@@ -511,7 +531,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
)
|
||||
any_queries[f'#_any_common_tags'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
|
||||
SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
AND tag IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -525,14 +545,16 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -551,15 +573,17 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$all_{attr}{i}' for i in range(len(all_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'claim.claim_hash__in#_all_{attr}'] = f"""
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{len(all_items)}=(
|
||||
SELECT count(*) FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -572,14 +596,16 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$not_{attr}{i}' for i in range(len(not_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'claim.claim_hash__not_in#_not_{attr}'] = f"""
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
NOT EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -24,16 +24,23 @@ DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
|
||||
TRENDING_LOG = True
|
||||
|
||||
|
||||
# Stubs
|
||||
def install(connection):
|
||||
"""
|
||||
Install the AR trending algorithm.
|
||||
"""
|
||||
check_trending_values(connection)
|
||||
|
||||
if TRENDING_LOG:
|
||||
f = open("trending_ar.log", "a")
|
||||
f.close()
|
||||
|
||||
# Stub
|
||||
CREATE_TREND_TABLE = ""
|
||||
|
||||
|
||||
def check_trending_values(connection):
|
||||
"""
|
||||
If the trending values appear to be based on the standard algorithm,
|
||||
If the trending values appear to be based on the zscore algorithm,
|
||||
reset them. This will allow resyncing from a standard snapshot.
|
||||
"""
|
||||
c = connection.cursor()
|
||||
@@ -53,9 +60,6 @@ def check_trending_values(connection):
|
||||
COMMIT;""")
|
||||
print("done.")
|
||||
|
||||
# Create the index
|
||||
c.execute("create index if not exists claim_id_only_idx on claim (claim_id);")
|
||||
|
||||
|
||||
def spike_height(trending_score, x, x_old, time_boost=1.0):
|
||||
"""
|
||||
@@ -71,8 +75,9 @@ def spike_height(trending_score, x, x_old, time_boost=1.0):
|
||||
|
||||
# Softened change in amount counts more for minnows
|
||||
if delta > 0.0:
|
||||
multiplier = 1.0/math.sqrt(x + 1.0)
|
||||
softened_change_in_amount *= multiplier
|
||||
if trending_score >= 0.0:
|
||||
multiplier = 0.1/((trending_score/time_boost + softened_change_in_amount) + 1.0)
|
||||
softened_change_in_amount *= multiplier
|
||||
else:
|
||||
softened_change_in_amount *= -1.0
|
||||
|
||||
@@ -106,14 +111,14 @@ class TrendingData:
|
||||
# Have all claims been read from db yet?
|
||||
self.initialised = False
|
||||
|
||||
def insert_claim_from_load(self, claim_id, trending_score, total_amount):
|
||||
def insert_claim_from_load(self, claim_hash, trending_score, total_amount):
|
||||
assert not self.initialised
|
||||
self.claims[claim_id] = {"trending_score": trending_score,
|
||||
"total_amount": total_amount,
|
||||
"changed": False}
|
||||
self.claims[claim_hash] = {"trending_score": trending_score,
|
||||
"total_amount": total_amount,
|
||||
"changed": False}
|
||||
|
||||
|
||||
def update_claim(self, claim_id, total_amount, time_boost=1.0):
|
||||
def update_claim(self, claim_hash, total_amount, time_boost=1.0):
|
||||
"""
|
||||
Update trending data for a claim, given its new total amount.
|
||||
"""
|
||||
@@ -121,8 +126,8 @@ class TrendingData:
|
||||
|
||||
# Extract existing total amount and trending score
|
||||
# or use starting values if the claim is new
|
||||
if claim_id in self.claims:
|
||||
old_state = copy.deepcopy(self.claims[claim_id])
|
||||
if claim_hash in self.claims:
|
||||
old_state = copy.deepcopy(self.claims[claim_hash])
|
||||
else:
|
||||
old_state = {"trending_score": 0.0,
|
||||
"total_amount": 0.0,
|
||||
@@ -138,9 +143,9 @@ class TrendingData:
|
||||
old_state["total_amount"],
|
||||
time_boost)
|
||||
trending_score = old_state["trending_score"] + spike
|
||||
self.claims[claim_id] = {"total_amount": total_amount,
|
||||
"trending_score": trending_score,
|
||||
"changed": True}
|
||||
self.claims[claim_hash] = {"total_amount": total_amount,
|
||||
"trending_score": trending_score,
|
||||
"changed": True}
|
||||
|
||||
|
||||
|
||||
@@ -170,13 +175,12 @@ def test_trending():
|
||||
# One global instance
|
||||
# pylint: disable=C0103
|
||||
trending_data = TrendingData()
|
||||
f = open("trending_ar.log", "w")
|
||||
f.close()
|
||||
|
||||
def run(db, height, final_height, recalculate_claim_hashes):
|
||||
|
||||
if height < final_height - 5*HALF_LIFE:
|
||||
trending_log("Skipping AR trending at block {h}.\n".format(h=height))
|
||||
return
|
||||
|
||||
start = time.time()
|
||||
|
||||
@@ -212,7 +216,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
|
||||
if not trending_data.initialised:
|
||||
# On fresh launch
|
||||
for row in db.execute("""
|
||||
SELECT claim_id, trending_mixed,
|
||||
SELECT claim_hash, trending_mixed,
|
||||
(amount + support_amount)
|
||||
AS total_amount
|
||||
FROM claim;
|
||||
@@ -221,7 +225,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
|
||||
trending_data.initialised = True
|
||||
else:
|
||||
for row in db.execute(f"""
|
||||
SELECT claim_id,
|
||||
SELECT claim_hash,
|
||||
(amount + support_amount)
|
||||
AS total_amount
|
||||
FROM claim
|
||||
@@ -248,7 +252,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
|
||||
|
||||
trending_log("{n} scores to write...".format(n=len(the_list)))
|
||||
|
||||
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_id=?;",
|
||||
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
|
||||
the_list)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
@@ -94,6 +94,7 @@ class SQLDB:
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -129,35 +130,42 @@ class SQLDB:
|
||||
-- used by any tag clouds
|
||||
create index if not exists tag_tag_idx on tag (tag, claim_hash);
|
||||
|
||||
-- common ORDER BY
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_trending_group_mixed_idx on claim (trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
|
||||
-- naked order bys (no filters)
|
||||
create unique index if not exists claim_release_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_idx on claim (trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
|
||||
|
||||
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
|
||||
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
|
||||
-- claim_type filter + order by
|
||||
create unique index if not exists claim_type_release_idx on claim (release_time, claim_type, claim_hash);
|
||||
create unique index if not exists claim_type_trending_idx on claim (trending_group, trending_mixed, claim_type, claim_hash);
|
||||
create unique index if not exists claim_type_effective_amount_idx on claim (effective_amount, claim_type, claim_hash);
|
||||
|
||||
create unique index if not exists channel_hash_release_time_idx on claim (channel_hash, release_time, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, effective_amount, claim_hash);
|
||||
-- stream_type filter + order by
|
||||
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
|
||||
create unique index if not exists stream_type_trending_idx on claim (stream_type, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists stream_type_effective_amount_idx on claim (stream_type, effective_amount, claim_hash);
|
||||
|
||||
create unique index if not exists filter_stream_duration_idx on claim (duration, trending_global, trending_mixed, claim_hash);
|
||||
-- channel_hash filter + order by
|
||||
create unique index if not exists channel_hash_release_idx on claim (channel_hash, release_time, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists channel_hash_effective_amount_idx on claim (channel_hash, effective_amount, claim_hash);
|
||||
|
||||
-- duration filter + order by
|
||||
create unique index if not exists duration_release_idx on claim (duration, release_time, claim_hash);
|
||||
create unique index if not exists duration_trending_idx on claim (duration, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists duration_effective_amount_idx on claim (duration, effective_amount, claim_hash);
|
||||
|
||||
-- fee_amount + order by
|
||||
create unique index if not exists fee_amount_release_idx on claim (fee_amount, release_time, claim_hash);
|
||||
create unique index if not exists fee_amount_trending_idx on claim (fee_amount, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists fee_amount_effective_amount_idx on claim (fee_amount, effective_amount, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
@@ -241,10 +249,10 @@ class SQLDB:
|
||||
streams, channels = {}, {}
|
||||
if channel_hashes:
|
||||
sql = query(
|
||||
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
|
||||
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
|
||||
'claim.reposted_claim_hash__is_not_null': 1,
|
||||
'claim.channel_hash__in': channel_hashes
|
||||
"SELECT repost.channel_hash, repost.reposted_claim_hash, target.claim_type "
|
||||
"FROM claim as repost JOIN claim AS target ON (target.claim_hash=repost.reposted_claim_hash)", **{
|
||||
'repost.reposted_claim_hash__is_not_null': 1,
|
||||
'repost.channel_hash__in': channel_hashes
|
||||
}
|
||||
)
|
||||
for blocked_claim in self.execute(*sql):
|
||||
@@ -360,6 +368,7 @@ class SQLDB:
|
||||
if isinstance(fee.amount, Decimal):
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
elif claim.is_repost:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['repost']
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
@@ -857,7 +866,7 @@ class LBRYLevelDB(LevelDB):
|
||||
super().__init__(*args, **kwargs)
|
||||
path = os.path.join(self.env.db_dir, 'claims.db')
|
||||
trending = []
|
||||
for algorithm_name in set(self.env.default('TRENDING_ALGORITHMS', 'zscore').split(' ')):
|
||||
for algorithm_name in self.env.trending_algorithms:
|
||||
if algorithm_name in TRENDING_ALGORITHMS:
|
||||
trending.append(TRENDING_ALGORITHMS[algorithm_name])
|
||||
self.sql = SQLDB(
|
||||
|
||||
@@ -37,6 +37,7 @@ class Env:
|
||||
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
@@ -72,16 +73,18 @@ class Env:
|
||||
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
|
||||
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
|
||||
# The electrum client takes the empty string as unspecified
|
||||
self.payment_address = self.default('PAYMENT_ADDRESS', '')
|
||||
self.donation_address = self.default('DONATION_ADDRESS', '')
|
||||
# Server limits to help prevent DoS
|
||||
self.max_send = self.integer('MAX_SEND', 1000000)
|
||||
self.max_receive = self.integer('MAX_RECEIVE', 1000000)
|
||||
self.max_subs = self.integer('MAX_SUBS', 250000)
|
||||
self.max_sessions = self.sane_max_sessions()
|
||||
self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
|
||||
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
|
||||
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
|
||||
self.description = self.default('DESCRIPTION', '')
|
||||
self.daily_fee = self.integer('DAILY_FEE', 0)
|
||||
self.daily_fee = self.string_amount('DAILY_FEE', '0')
|
||||
|
||||
# Identities
|
||||
clearnet_identity = self.clearnet_identity()
|
||||
@@ -107,6 +110,14 @@ class Env:
|
||||
raise cls.Error(f'required envvar {envvar} not set')
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def string_amount(cls, envvar, default):
|
||||
value = environ.get(envvar, default)
|
||||
amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?")
|
||||
if len(value) > 0 and not amount_pattern.fullmatch(value):
|
||||
raise cls.Error(f'{value} is not a valid amount for {envvar}')
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def integer(cls, envvar, default):
|
||||
value = environ.get(envvar)
|
||||
|
||||
@@ -20,7 +20,7 @@ from collections import namedtuple
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from struct import pack, unpack
|
||||
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
import attr
|
||||
|
||||
from lbry.wallet.server import util
|
||||
@@ -63,6 +63,7 @@ class LevelDB:
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.env = env
|
||||
self.coin = env.coin
|
||||
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
|
||||
|
||||
# Setup block header size handlers
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
@@ -139,6 +140,7 @@ class LevelDB:
|
||||
def close(self):
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.executor.shutdown(wait=True)
|
||||
|
||||
async def open_for_compacting(self):
|
||||
await self._open_dbs(True, True)
|
||||
@@ -406,7 +408,7 @@ class LevelDB:
|
||||
return self.headers_file.read(offset, size), disk_count
|
||||
return b'', 0
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(None, read_headers)
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
|
||||
|
||||
def fs_tx_hash(self, tx_num):
|
||||
"""Return a par (tx_hash, tx_height) for the given tx number.
|
||||
@@ -445,7 +447,7 @@ class LevelDB:
|
||||
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
|
||||
|
||||
while True:
|
||||
history = await asyncio.get_event_loop().run_in_executor(None, read_history)
|
||||
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
|
||||
if all(hash is not None for hash, height in history):
|
||||
return history
|
||||
self.logger.warning(f'limited_history: tx hash '
|
||||
@@ -611,7 +613,7 @@ class LevelDB:
|
||||
return utxos
|
||||
|
||||
while True:
|
||||
utxos = await asyncio.get_event_loop().run_in_executor(None, read_utxos)
|
||||
utxos = await asyncio.get_event_loop().run_in_executor(self.executor, read_utxos)
|
||||
if all(utxo.tx_hash is not None for utxo in utxos):
|
||||
return utxos
|
||||
self.logger.warning(f'all_utxos: tx hash not '
|
||||
@@ -664,5 +666,5 @@ class LevelDB:
|
||||
return hashX, value
|
||||
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
|
||||
|
||||
hashX_pairs = await asyncio.get_event_loop().run_in_executor(None, lookup_hashXs)
|
||||
return await asyncio.get_event_loop().run_in_executor(None, lookup_utxos, hashX_pairs)
|
||||
hashX_pairs = await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs)
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_utxos, hashX_pairs)
|
||||
@@ -6,12 +6,13 @@
|
||||
# and warranty status of this software.
|
||||
|
||||
"""Mempool handling."""
|
||||
|
||||
import os
|
||||
import asyncio
|
||||
import itertools
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
import attr
|
||||
|
||||
@@ -105,14 +106,15 @@ class MemPool:
|
||||
# Prevents mempool refreshes during fee histogram calculation
|
||||
self.lock = asyncio.Lock()
|
||||
self.wakeup = asyncio.Event()
|
||||
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
|
||||
|
||||
async def _logging(self, synchronized_event):
|
||||
"""Print regular logs of mempool stats."""
|
||||
self.logger.info('beginning processing of daemon mempool. '
|
||||
'This can take some time...')
|
||||
start = time.time()
|
||||
start = time.perf_counter()
|
||||
await synchronized_event.wait()
|
||||
elapsed = time.time() - start
|
||||
elapsed = time.perf_counter() - start
|
||||
self.logger.info(f'synced in {elapsed:.2f}s')
|
||||
while True:
|
||||
self.logger.info(f'{len(self.txs):,d} txs '
|
||||
@@ -121,12 +123,15 @@ class MemPool:
|
||||
await synchronized_event.wait()
|
||||
|
||||
async def _refresh_histogram(self, synchronized_event):
|
||||
while True:
|
||||
await synchronized_event.wait()
|
||||
async with self.lock:
|
||||
# Threaded as can be expensive
|
||||
await asyncio.get_event_loop().run_in_executor(None, self._update_histogram, 100_000)
|
||||
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
|
||||
try:
|
||||
while True:
|
||||
await synchronized_event.wait()
|
||||
async with self.lock:
|
||||
# Threaded as can be expensive
|
||||
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
|
||||
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
|
||||
finally:
|
||||
self.executor.shutdown(wait=True)
|
||||
|
||||
def _update_histogram(self, bin_size):
|
||||
# Build a histogram by fee rate
|
||||
@@ -289,7 +294,7 @@ class MemPool:
|
||||
return txs
|
||||
|
||||
# Thread this potentially slow operation so as not to block
|
||||
tx_map = await asyncio.get_event_loop().run_in_executor(None, deserialize_txs)
|
||||
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
|
||||
|
||||
# Determine all prevouts not in the mempool, and fetch the
|
||||
# UTXO information from the database. Failed prevout lookups
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from aiohttp import web
|
||||
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
|
||||
from lbry import __version__ as version
|
||||
@@ -6,7 +7,7 @@ from lbry.wallet.server import util
|
||||
import lbry.wallet.server.version as wallet_server_version
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
CPU_COUNT = f"{os.cpu_count()}"
|
||||
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
|
||||
VERSION_INFO.info({
|
||||
'build': BUILD,
|
||||
@@ -14,16 +15,17 @@ VERSION_INFO.info({
|
||||
"docker_tag": DOCKER_TAG,
|
||||
'version': version,
|
||||
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
|
||||
"cpu_count": CPU_COUNT
|
||||
})
|
||||
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE)
|
||||
|
||||
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version", ))
|
||||
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method",))
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method",))
|
||||
labelnames=("method", "version"))
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method",))
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method",))
|
||||
labelnames=("method", "version"))
|
||||
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
|
||||
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
|
||||
@@ -45,6 +47,10 @@ CLIENT_VERSIONS = Counter(
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
BLOCK_COUNT = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
|
||||
@@ -12,7 +12,6 @@ import asyncio
|
||||
import logging
|
||||
import itertools
|
||||
import collections
|
||||
from functools import lru_cache
|
||||
|
||||
from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
@@ -31,6 +30,7 @@ from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
|
||||
from lbry.wallet.rpc.framing import NewlineFramer
|
||||
import lbry.wallet.server.version as VERSION
|
||||
|
||||
from lbry.wallet.rpc import (
|
||||
@@ -327,7 +327,7 @@ class SessionManager:
|
||||
return [(session.session_id,
|
||||
session.flags(),
|
||||
session.peer_address_str(for_log=for_log),
|
||||
session.client,
|
||||
session.client_version,
|
||||
session.protocol_version_string(),
|
||||
session.count_pending_items(),
|
||||
session.txs_sent,
|
||||
@@ -622,6 +622,7 @@ class SessionBase(RPCSession):
|
||||
|
||||
def __init__(self, session_mgr, db, mempool, peer_mgr, kind):
|
||||
connection = JSONRPCConnection(JSONRPCAutoDetect)
|
||||
self.env = session_mgr.env
|
||||
super().__init__(connection=connection)
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.session_mgr = session_mgr
|
||||
@@ -629,9 +630,7 @@ class SessionBase(RPCSession):
|
||||
self.mempool = mempool
|
||||
self.peer_mgr = peer_mgr
|
||||
self.kind = kind # 'RPC', 'TCP' etc.
|
||||
self.env = session_mgr.env
|
||||
self.coin = self.env.coin
|
||||
self.client = 'unknown'
|
||||
self.anon_logs = self.env.anon_logs
|
||||
self.txs_sent = 0
|
||||
self.log_me = False
|
||||
@@ -643,6 +642,9 @@ class SessionBase(RPCSession):
|
||||
async def notify(self, touched, height_changed):
|
||||
pass
|
||||
|
||||
def default_framer(self):
|
||||
return NewlineFramer(self.env.max_receive)
|
||||
|
||||
def peer_address_str(self, *, for_log=True):
|
||||
"""Returns the peer's IP address and port as a human-readable
|
||||
string, respecting anon logs if the output is for a log."""
|
||||
@@ -675,15 +677,16 @@ class SessionBase(RPCSession):
|
||||
context = {'conn_id': f'{self.session_id}'}
|
||||
self.logger = util.ConnectionLogger(self.logger, context)
|
||||
self.group = self.session_mgr.add_session(self)
|
||||
SESSIONS_COUNT.inc()
|
||||
self.logger.info(f'{self.kind} {self.peer_address_str()}, '
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
peer_addr_str = self.peer_address_str()
|
||||
self.logger.info(f'{self.kind} {peer_addr_str}, '
|
||||
f'{self.session_mgr.session_count():,d} total')
|
||||
|
||||
def connection_lost(self, exc):
|
||||
"""Handle client disconnection."""
|
||||
super().connection_lost(exc)
|
||||
self.session_mgr.remove_session(self)
|
||||
SESSIONS_COUNT.dec()
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
msg = ''
|
||||
if not self._can_send.is_set():
|
||||
msg += ' whilst paused'
|
||||
@@ -707,7 +710,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle an incoming request. ElectrumX doesn't receive
|
||||
notifications from client sessions.
|
||||
"""
|
||||
REQUESTS_COUNT.labels(method=request.method).inc()
|
||||
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
|
||||
if isinstance(request, Request):
|
||||
handler = self.request_handlers.get(request.method)
|
||||
handler = partial(handler, self)
|
||||
@@ -801,6 +804,7 @@ class LBRYElectrumX(SessionBase):
|
||||
'blockchain.transaction.get_merkle': cls.transaction_merkle,
|
||||
'server.add_peer': cls.add_peer,
|
||||
'server.banner': cls.banner,
|
||||
'server.payment_address': cls.payment_address,
|
||||
'server.donation_address': cls.donation_address,
|
||||
'server.features': cls.server_features_async,
|
||||
'server.peers.subscribe': cls.peers_subscribe,
|
||||
@@ -838,12 +842,6 @@ class LBRYElectrumX(SessionBase):
|
||||
self.bp: LBRYBlockProcessor = self.session_mgr.bp
|
||||
self.db: LBRYLevelDB = self.bp.db
|
||||
|
||||
@lru_cache(1000)
|
||||
async def _cached_block_headers(_start_height, _count, _b64, _max_size, _cp_height):
|
||||
return await self._block_headers(_start_height, _count, _b64, _max_size, _cp_height)
|
||||
|
||||
self._cached_block_headers = _cached_block_headers
|
||||
|
||||
@classmethod
|
||||
def protocol_min_max_strings(cls):
|
||||
return [util.version_string(ver)
|
||||
@@ -861,9 +859,11 @@ class LBRYElectrumX(SessionBase):
|
||||
'protocol_max': max_str,
|
||||
'genesis_hash': env.coin.GENESIS_HASH,
|
||||
'description': env.description,
|
||||
'payment_address': env.donation_address,
|
||||
'payment_address': env.payment_address,
|
||||
'donation_address': env.donation_address,
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
'trending_algorithm': env.trending_algorithms[0]
|
||||
}
|
||||
|
||||
async def server_features_async(self):
|
||||
@@ -1306,20 +1306,6 @@ class LBRYElectrumX(SessionBase):
|
||||
height: the header's height"""
|
||||
return await self.block_header(height)
|
||||
|
||||
async def _block_headers(self, start_height, count, b64, max_size, cp_height):
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'count': count,
|
||||
'max': max_size
|
||||
}
|
||||
if count and cp_height:
|
||||
last_height = start_height + count - 1
|
||||
result.update(await self._merkle_proof(cp_height, last_height))
|
||||
return result
|
||||
|
||||
async def block_headers(self, start_height, count, cp_height=0, b64=False):
|
||||
"""Return count concatenated block headers as hex for the main chain;
|
||||
starting at start_height.
|
||||
@@ -1333,10 +1319,18 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
max_size = self.MAX_CHUNK_SIZE
|
||||
count = min(count, max_size)
|
||||
|
||||
if count + start_height < self.db.db_height:
|
||||
return await self._cached_block_headers(start_height, count, b64, max_size, cp_height)
|
||||
return await self._block_headers(start_height, count, b64, max_size, cp_height)
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'count': count,
|
||||
'max': max_size
|
||||
}
|
||||
if count and cp_height:
|
||||
last_height = start_height + count - 1
|
||||
result.update(await self._merkle_proof(cp_height, last_height))
|
||||
return result
|
||||
|
||||
async def block_get_chunk(self, index):
|
||||
"""Return a chunk of block headers as a hexadecimal string.
|
||||
@@ -1375,11 +1369,16 @@ class LBRYElectrumX(SessionBase):
|
||||
('$SERVER_VERSION', self.version),
|
||||
('$DAEMON_VERSION', daemon_version),
|
||||
('$DAEMON_SUBVERSION', network_info['subversion']),
|
||||
('$PAYMENT_ADDRESS', self.env.payment_address),
|
||||
('$DONATION_ADDRESS', self.env.donation_address),
|
||||
]:
|
||||
banner = banner.replace(*pair)
|
||||
return banner
|
||||
|
||||
async def payment_address(self):
|
||||
"""Return the payment address as a string, empty if there is none."""
|
||||
return self.env.payment_address
|
||||
|
||||
async def donation_address(self):
|
||||
"""Return the donation address as a string, empty if there is none."""
|
||||
return self.env.donation_address
|
||||
@@ -1429,6 +1428,7 @@ class LBRYElectrumX(SessionBase):
|
||||
client_name: a string identifying the client
|
||||
protocol_version: the protocol version spoken by the client
|
||||
"""
|
||||
|
||||
if self.sv_seen and self.protocol_tuple >= (1, 4):
|
||||
raise RPCError(BAD_REQUEST, f'server.version already sent')
|
||||
self.sv_seen = True
|
||||
@@ -1440,8 +1440,11 @@ class LBRYElectrumX(SessionBase):
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported client: {client_name}')
|
||||
self.client = client_name[:17]
|
||||
CLIENT_VERSIONS.labels(version=str(client_name)).inc()
|
||||
if self.client_version != client_name[:17]:
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
self.client_version = client_name[:17]
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
CLIENT_VERSIONS.labels(version=self.client_version).inc()
|
||||
|
||||
# Find the highest common protocol version. Disconnect if
|
||||
# that protocol version in unsupported.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from lbry.error import (
|
||||
ServerPaymentFeeAboveMaxAllowedError,
|
||||
ServerPaymentInvalidAddressError,
|
||||
ServerPaymentWalletLockedError
|
||||
)
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.transaction import Output, Transaction
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WalletServerPayer:
|
||||
def __init__(self, payment_period=24 * 60 * 60, max_fee='1.0', analytics_manager=None):
|
||||
self.ledger = None
|
||||
self.wallet = None
|
||||
self.running = False
|
||||
self.task = None
|
||||
self.payment_period = payment_period
|
||||
self.analytics_manager = analytics_manager
|
||||
self.max_fee = max_fee
|
||||
self._on_payment_controller = StreamController()
|
||||
self.on_payment = self._on_payment_controller.stream
|
||||
self.on_payment.listen(None, on_error=lambda e: logging.warning(e.args[0]))
|
||||
|
||||
async def pay(self):
|
||||
while self.running:
|
||||
await asyncio.sleep(self.payment_period)
|
||||
features = await self.ledger.network.retriable_call(self.ledger.network.get_server_features)
|
||||
address = features['payment_address']
|
||||
amount = str(features['daily_fee'])
|
||||
if not address or not amount:
|
||||
continue
|
||||
|
||||
if not self.ledger.is_valid_address(address):
|
||||
self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address))
|
||||
continue
|
||||
|
||||
if self.wallet.is_locked:
|
||||
self._on_payment_controller.add_error(ServerPaymentWalletLockedError())
|
||||
continue
|
||||
|
||||
amount = lbc_to_dewies(features['daily_fee']) # check that this is in lbc and not dewies
|
||||
limit = lbc_to_dewies(self.max_fee)
|
||||
if amount > limit:
|
||||
self._on_payment_controller.add_error(
|
||||
ServerPaymentFeeAboveMaxAllowedError(features['daily_fee'], self.max_fee)
|
||||
)
|
||||
continue
|
||||
|
||||
tx = await Transaction.create(
|
||||
[],
|
||||
[Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))],
|
||||
self.wallet.get_accounts_or_all(None),
|
||||
self.wallet.get_account_or_default(None)
|
||||
)
|
||||
|
||||
await self.ledger.broadcast(tx)
|
||||
if self.analytics_manager:
|
||||
await self.analytics_manager.send_credits_sent()
|
||||
self._on_payment_controller.add(tx)
|
||||
|
||||
async def start(self, ledger=None, wallet=None):
|
||||
if lbc_to_dewies(self.max_fee) < 1:
|
||||
return
|
||||
self.ledger = ledger
|
||||
self.wallet = wallet
|
||||
self.running = True
|
||||
self.task = asyncio.ensure_future(self.pay())
|
||||
self.task.add_done_callback(lambda _: log.info("Stopping wallet server payments."))
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self.task.cancel()
|
||||
@@ -26,8 +26,8 @@ async def process_video(analyzer, video_file):
|
||||
try:
|
||||
await analyzer.verify_or_repair(True, False, video_file)
|
||||
print("No concerns. Ship it!")
|
||||
except FileNotFoundError as e:
|
||||
print(str(e))
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print("Analysis failed.", str(e))
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
transcode = input("Would you like to make a repaired clone now? [y/N] ")
|
||||
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# usage: update_dev_wallet_server.sh <host to update>
|
||||
TARGET_HOST=$1
|
||||
|
||||
SCRIPTS_DIR=`dirname $0`
|
||||
LBRY_DIR=`dirname $SCRIPTS_DIR`
|
||||
|
||||
# build the image
|
||||
docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR
|
||||
IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"`
|
||||
|
||||
# push the image to the server
|
||||
ssh $TARGET_HOST docker image prune --force
|
||||
docker save $IMAGE | ssh $TARGET_HOST docker load
|
||||
ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development
|
||||
|
||||
# restart the wallet server
|
||||
ssh $TARGET_HOST docker-compose down
|
||||
ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d
|
||||
@@ -333,7 +333,7 @@ class Examples(CommandTestCase):
|
||||
)
|
||||
|
||||
reply = await r(
|
||||
'You can r',
|
||||
'Use the parent_id param to make replies',
|
||||
'comment', 'create',
|
||||
'--comment="I have photographic evidence confirming Sasquatch exists"',
|
||||
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
|
||||
|
||||
+6
-2
@@ -124,7 +124,11 @@ def release(args):
|
||||
|
||||
current_version = Version.from_content(version_file)
|
||||
print(f'Current Version: {current_version}')
|
||||
new_version = current_version.increment(args.action)
|
||||
|
||||
if args.action == 'current':
|
||||
new_version = current_version
|
||||
else:
|
||||
new_version = current_version.increment(args.action)
|
||||
print(f' New Version: {new_version}')
|
||||
|
||||
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
|
||||
@@ -241,7 +245,7 @@ def main():
|
||||
parser.add_argument("--confirm", default=False, action="store_true",
|
||||
help="without this flag, it will only print what it will do but will not actually do it")
|
||||
parser.add_argument("--start-tag", help="custom starting tag for changelog generation")
|
||||
parser.add_argument("action", choices=['test', 'major', 'minor', 'micro'])
|
||||
parser.add_argument("action", choices=['test', 'current', 'major', 'minor', 'micro'])
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.action == "test":
|
||||
|
||||
@@ -12,7 +12,7 @@ async def main():
|
||||
try:
|
||||
hostname = sys.argv[1]
|
||||
except IndexError:
|
||||
hostname = 'spv1.lbry.com'
|
||||
hostname = 'spv11.lbry.com'
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
client = ClientSession(network=None, server=(hostname, 50001))
|
||||
|
||||
@@ -32,6 +32,7 @@ disable=
|
||||
too-many-branches,
|
||||
too-many-arguments,
|
||||
too-many-statements,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-instance-attributes,
|
||||
protected-access,
|
||||
|
||||
@@ -40,7 +40,7 @@ setup(
|
||||
'colorama==0.3.7',
|
||||
'distro==1.4.0',
|
||||
'base58==1.0.0',
|
||||
'cffi==1.12.1',
|
||||
'cffi==1.13.2',
|
||||
'cryptography==2.5',
|
||||
'protobuf==3.6.1',
|
||||
'msgpack==0.6.1',
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import os.path
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
from lbry.error import InsufficientFundsError
|
||||
|
||||
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
|
||||
@@ -73,6 +73,19 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
(result['txid'], result['claim_id'])
|
||||
)
|
||||
|
||||
async def test_disconnect_on_memory_error(self):
|
||||
claim_ids = [
|
||||
'0000000000000000000000000000000000000000',
|
||||
] * 23828
|
||||
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
|
||||
|
||||
# 23829 claim ids makes the request just large enough
|
||||
claim_ids = [
|
||||
'0000000000000000000000000000000000000000',
|
||||
] * 23829
|
||||
with self.assertRaises(ConnectionResetError):
|
||||
await self.claim_search(claim_ids=claim_ids)
|
||||
|
||||
async def test_basic_claim_search(self):
|
||||
await self.create_channel()
|
||||
channel_txo = self.channel['outputs'][0]
|
||||
@@ -250,6 +263,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
|
||||
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
|
||||
chan3_id = self.get_claim_id(await self.channel_create('@chan3'))
|
||||
chan4 = await self.channel_create('@chan4', '0.1')
|
||||
|
||||
claim1 = await self.stream_create('claim1')
|
||||
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
|
||||
@@ -272,6 +286,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await match([claim6, claim5, claim4, claim1], not_channel_ids=[chan1_id], claim_type='stream')
|
||||
await match([claim6, claim3, claim2, claim1], not_channel_ids=[chan2_id], claim_type='stream')
|
||||
await match([claim6, claim1], not_channel_ids=[chan1_id, chan2_id], claim_type='stream')
|
||||
await match([claim6, claim1, chan4], not_channel_ids=[chan1_id, chan2_id])
|
||||
|
||||
# not_channel_ids + valid_channel_signature
|
||||
await match([claim5, claim4, claim3, claim2, claim1],
|
||||
@@ -309,12 +324,14 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
@@ -322,7 +339,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
|
||||
# stream_type
|
||||
# media_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
@@ -384,7 +401,25 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
|
||||
class ClaimCommands(ClaimTestCase):
|
||||
|
||||
async def test_claim_list_type_filtering(self):
|
||||
await self.channel_create()
|
||||
await self.stream_create()
|
||||
|
||||
r = await self.claim_list(claim_type='channel')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
|
||||
r = await self.claim_list(claim_type='stream')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('stream', r[0]['value_type'])
|
||||
|
||||
r = await self.claim_list(claim_type=['stream', 'channel'])
|
||||
self.assertEqual(2, len(r))
|
||||
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
|
||||
|
||||
async def test_claim_stream_channel_list_with_resolve(self):
|
||||
self.assertListEqual([], await self.claim_list(resolve=True))
|
||||
|
||||
await self.channel_create()
|
||||
await self.stream_create()
|
||||
|
||||
@@ -405,10 +440,10 @@ class ClaimCommands(ClaimTestCase):
|
||||
await self.ledger.wait(channel_tx)
|
||||
|
||||
r = await self.claim_list(resolve=True)
|
||||
self.assertEqual('not_found', r[0]['meta']['error']['name'])
|
||||
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
|
||||
self.assertTrue(r[1]['meta']['is_controlling'])
|
||||
r = await self.channel_list(resolve=True)
|
||||
self.assertEqual('not_found', r[0]['meta']['error']['name'])
|
||||
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
|
||||
self.assertTrue(r[1]['meta']['is_controlling'])
|
||||
|
||||
# confirm it
|
||||
@@ -430,10 +465,10 @@ class ClaimCommands(ClaimTestCase):
|
||||
await self.ledger.wait(stream_tx)
|
||||
|
||||
r = await self.claim_list(resolve=True)
|
||||
self.assertEqual('not_found', r[0]['meta']['error']['name'])
|
||||
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
|
||||
self.assertTrue(r[1]['meta']['is_controlling'])
|
||||
r = await self.stream_list(resolve=True)
|
||||
self.assertEqual('not_found', r[0]['meta']['error']['name'])
|
||||
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
|
||||
self.assertTrue(r[1]['meta']['is_controlling'])
|
||||
|
||||
# confirm it
|
||||
@@ -453,6 +488,9 @@ class ClaimCommands(ClaimTestCase):
|
||||
self.assertTrue(r[0]['meta']['is_controlling'])
|
||||
self.assertTrue(r[1]['meta']['is_controlling'])
|
||||
|
||||
# check that metadata is transfered
|
||||
self.assertTrue(r[0]['is_mine'])
|
||||
|
||||
|
||||
class ChannelCommands(CommandTestCase):
|
||||
|
||||
@@ -794,8 +832,11 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
|
||||
|
||||
async def test_repost(self):
|
||||
sql = self.conductor.spv_node.server.bp.sql
|
||||
sql.execute(sql.TAG_INDEXES)
|
||||
|
||||
await self.channel_create('@goodies', '1.0')
|
||||
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies')
|
||||
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies', tags=['foo', 'gaming'])
|
||||
claim_id = self.get_claim_id(tx)
|
||||
|
||||
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0)
|
||||
@@ -806,6 +847,16 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1)
|
||||
self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
|
||||
|
||||
# tags are inherited (non-common / indexed tags)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['foo'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['foo'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['foo'], claim_type=['stream', 'repost']), 0)
|
||||
# "common" / indexed tags work too
|
||||
self.assertIn('gaming', sql.TAG_INDEXES) # if this breaks, next test doesn't make sense
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(any_tags=['gaming'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(all_tags=['gaming'], claim_type=['stream', 'repost']), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(not_tags=['gaming'], claim_type=['stream', 'repost']), 0)
|
||||
|
||||
await self.channel_create('@reposting-goodies', '1.0')
|
||||
await self.stream_repost(claim_id, 'repost-on-channel', '1.1', channel_name='@reposting-goodies')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 2)
|
||||
@@ -825,57 +876,114 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(search['reposted_claim']['meta']['reposted'], 2)
|
||||
self.assertEqual(search['reposted_claim']['signing_channel']['name'], '@goodies')
|
||||
|
||||
resolved = await self.resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
|
||||
resolved = await self.out(
|
||||
self.daemon.jsonrpc_resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
|
||||
)
|
||||
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
|
||||
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
|
||||
|
||||
async def test_filtering_channels_for_removing_content(self):
|
||||
await self.channel_create('@some_channel', '1.0')
|
||||
await self.stream_create('good_content', '1.1', channel_name='@some_channel', tags=['good'])
|
||||
await self.channel_create('@some_channel', '0.1')
|
||||
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
|
||||
bad_content_id = self.get_claim_id(
|
||||
await self.stream_create('bad_content', '1.1', channel_name='@some_channel', tags=['bad'])
|
||||
await self.stream_create('bad_content', '0.1', channel_name='@some_channel', tags=['bad'])
|
||||
)
|
||||
filtering_channel_id = self.get_claim_id(
|
||||
await self.channel_create('@filtering', '1.0')
|
||||
await self.channel_create('@filtering', '0.1')
|
||||
)
|
||||
self.conductor.spv_node.server.db.sql.filtering_channel_hashes.add(
|
||||
unhexlify(filtering_channel_id)[::-1]
|
||||
)
|
||||
await self.stream_repost(bad_content_id, 'filter1', '1.1', channel_name='@filtering')
|
||||
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.filtered_streams))
|
||||
await self.stream_repost(bad_content_id, 'filter1', '0.1', channel_name='@filtering')
|
||||
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.filtered_streams))
|
||||
|
||||
# search for blocked content directly
|
||||
# search for filtered content directly
|
||||
result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content'))
|
||||
blocked = result['blocked']
|
||||
self.assertEqual([], result['items'])
|
||||
self.assertEqual({"channels": {filtering_channel_id: 1}, "total": 1}, result['blocked'])
|
||||
self.assertEqual(1, blocked['total'])
|
||||
self.assertEqual(1, len(blocked['channels']))
|
||||
self.assertEqual(1, blocked['channels'][0]['blocked'])
|
||||
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
|
||||
|
||||
# search channel containing blocked content
|
||||
# search inside channel containing filtered content
|
||||
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel'))
|
||||
filtered = result['blocked']
|
||||
self.assertEqual(1, len(result['items']))
|
||||
self.assertEqual({"channels": {filtering_channel_id: 1}, "total": 1}, result['blocked'])
|
||||
self.assertEqual(1, filtered['total'])
|
||||
self.assertEqual(1, len(filtered['channels']))
|
||||
self.assertEqual(1, filtered['channels'][0]['blocked'])
|
||||
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
|
||||
|
||||
# content was filtered by not_tag before censoring
|
||||
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"]))
|
||||
self.assertEqual(0, len(result['items']))
|
||||
self.assertEqual({"channels": {}, "total": 0}, result['blocked'])
|
||||
self.assertEqual({"channels": [], "total": 0}, result['blocked'])
|
||||
|
||||
# filtered content can still be resolved
|
||||
result = await self.resolve('lbry://@some_channel/bad_content')
|
||||
self.assertEqual(bad_content_id, result['claim_id'])
|
||||
|
||||
blocking_channel_id = self.get_claim_id(
|
||||
await self.channel_create('@blocking', '1.0')
|
||||
await self.channel_create('@blocking', '0.1')
|
||||
)
|
||||
self.conductor.spv_node.server.db.sql.blocking_channel_hashes.add(
|
||||
unhexlify(blocking_channel_id)[::-1]
|
||||
)
|
||||
|
||||
# filtered content can still be resolved
|
||||
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
|
||||
self.assertEqual(bad_content_id, result['lbry://@some_channel/bad_content']['claim_id'])
|
||||
|
||||
await self.stream_repost(bad_content_id, 'block1', '1.1', channel_name='@blocking')
|
||||
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.blocked_streams))
|
||||
await self.stream_repost(bad_content_id, 'block1', '0.1', channel_name='@blocking')
|
||||
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.blocked_streams))
|
||||
|
||||
# blocked content is not resolveable
|
||||
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
|
||||
error = result['lbry://@some_channel/bad_content']['error']
|
||||
self.assertTrue(error['name'], 'blocked')
|
||||
error = (await self.resolve('lbry://@some_channel/bad_content'))['error']
|
||||
self.assertEqual(error['name'], 'BLOCKED')
|
||||
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
|
||||
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
|
||||
|
||||
# a filtered/blocked channel impacts all content inside it
|
||||
bad_channel_id = self.get_claim_id(
|
||||
await self.channel_create('@bad_channel', '0.1', tags=['bad-stuff'])
|
||||
)
|
||||
worse_content_id = self.get_claim_id(
|
||||
await self.stream_create('worse_content', '0.1', channel_name='@bad_channel', tags=['bad-stuff'])
|
||||
)
|
||||
|
||||
# check search before filtering channel
|
||||
result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height']))
|
||||
self.assertEqual(2, result['total_items'])
|
||||
self.assertEqual('worse_content', result['items'][0]['name'])
|
||||
self.assertEqual('@bad_channel', result['items'][1]['name'])
|
||||
|
||||
# filter channel out
|
||||
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.filtered_channels))
|
||||
await self.stream_repost(bad_channel_id, 'filter2', '0.1', channel_name='@filtering')
|
||||
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.filtered_channels))
|
||||
|
||||
# same claim search as previous now returns 0 results
|
||||
result = await self.out(self.daemon.jsonrpc_claim_search(any_tags=['bad-stuff'], order_by=['height']))
|
||||
filtered = result['blocked']
|
||||
self.assertEqual(0, len(result['items']))
|
||||
self.assertEqual(3, filtered['total'])
|
||||
self.assertEqual(1, len(filtered['channels']))
|
||||
self.assertEqual(3, filtered['channels'][0]['blocked'])
|
||||
self.assertTrue(filtered['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
|
||||
|
||||
# filtered channel should still resolve
|
||||
result = await self.resolve('lbry://@bad_channel')
|
||||
self.assertEqual(bad_channel_id, result['claim_id'])
|
||||
result = await self.resolve('lbry://@bad_channel/worse_content')
|
||||
self.assertEqual(worse_content_id, result['claim_id'])
|
||||
|
||||
# block channel
|
||||
self.assertEqual(0, len(self.conductor.spv_node.server.db.sql.blocked_channels))
|
||||
await self.stream_repost(bad_channel_id, 'block2', '0.1', channel_name='@blocking')
|
||||
self.assertEqual(1, len(self.conductor.spv_node.server.db.sql.blocked_channels))
|
||||
|
||||
# channel, claim in channel or claim individually no longer resolve
|
||||
self.assertEqual((await self.resolve('lbry://@bad_channel'))['error']['name'], 'BLOCKED')
|
||||
self.assertEqual((await self.resolve('lbry://worse_content'))['error']['name'], 'BLOCKED')
|
||||
self.assertEqual((await self.resolve('lbry://@bad_channel/worse_content'))['error']['name'], 'BLOCKED')
|
||||
|
||||
async def test_publish_updates_file_list(self):
|
||||
tx = await self.stream_create(title='created')
|
||||
@@ -1332,15 +1440,14 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
|
||||
r = await self.resolve('lbry://@abc/foo')
|
||||
self.assertEqual(
|
||||
r['lbry://@abc/foo']['claim_id'],
|
||||
r['claim_id'],
|
||||
self.get_claim_id(tx3)
|
||||
)
|
||||
|
||||
# publishing again clears channel
|
||||
tx4 = await self.publish('foo', languages='uk-UA', tags=['Anime', 'anime '])
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 2)
|
||||
r = await self.resolve('lbry://foo')
|
||||
claim = r['lbry://foo']
|
||||
claim = await self.resolve('lbry://foo')
|
||||
self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
|
||||
self.assertNotIn('signing_channel', claim)
|
||||
self.assertEqual(claim['value']['languages'], ['uk-UA'])
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
|
||||
import lbry
|
||||
@@ -29,15 +28,22 @@ class NetworkTests(IntegrationTestCase):
|
||||
'pruning': None,
|
||||
'description': '',
|
||||
'payment_address': '',
|
||||
'daily_fee': 0,
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
'donation_address': '',
|
||||
'daily_fee': '0',
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
address = (await self.account.get_addresses(limit=1))[0]
|
||||
os.environ.update({
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'DONATION_ADDRESS': address,
|
||||
'DAILY_FEE': '42'})
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
await self.conductor.spv_node.start(
|
||||
self.conductor.blockchain_node,
|
||||
extraconf={
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'PAYMENT_ADDRESS': payment_address,
|
||||
'DONATION_ADDRESS': donation_address,
|
||||
'DAILY_FEE': '42'
|
||||
}
|
||||
)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
@@ -47,9 +53,12 @@ class NetworkTests(IntegrationTestCase):
|
||||
'protocol_min': '0.54.0',
|
||||
'pruning': None,
|
||||
'description': 'Fastest server in the west.',
|
||||
'payment_address': address,
|
||||
'daily_fee': 42,
|
||||
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
|
||||
'payment_address': payment_address,
|
||||
'donation_address': donation_address,
|
||||
'daily_fee': '42',
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
|
||||
|
||||
class ReconnectTests(IntegrationTestCase):
|
||||
|
||||
@@ -148,7 +148,7 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
|
||||
url = result[0]['canonical_url']
|
||||
resolve = await self.resolve(url)
|
||||
self.assertEqual(result[0]['claim_id'], resolve[url]['purchase_receipt']['claim_id'])
|
||||
self.assertEqual(result[0]['claim_id'], resolve['purchase_receipt']['claim_id'])
|
||||
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
|
||||
await self.daemon.jsonrpc_get('lbry://a')
|
||||
|
||||
@@ -12,10 +12,10 @@ from lbry.crypto.base58 import Base58
|
||||
class BaseResolveTestCase(CommandTestCase):
|
||||
|
||||
async def assertResolvesToClaimId(self, name, claim_id):
|
||||
other = (await self.resolve(name))[name]
|
||||
other = await self.resolve(name)
|
||||
if claim_id is None:
|
||||
self.assertIn('error', other)
|
||||
self.assertEqual(other['error']['name'], 'not_found')
|
||||
self.assertEqual(other['error']['name'], 'NOT_FOUND')
|
||||
else:
|
||||
self.assertEqual(claim_id, other['claim_id'])
|
||||
|
||||
@@ -29,23 +29,20 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
|
||||
# resolving a channel @abc
|
||||
response = await self.resolve('lbry://@abc')
|
||||
self.assertSetEqual({'lbry://@abc'}, set(response))
|
||||
self.assertEqual(response['lbry://@abc']['name'], '@abc')
|
||||
self.assertEqual(response['lbry://@abc']['value_type'], 'channel')
|
||||
self.assertEqual(response['lbry://@abc']['meta']['claims_in_channel'], 0)
|
||||
self.assertEqual(response['name'], '@abc')
|
||||
self.assertEqual(response['value_type'], 'channel')
|
||||
self.assertEqual(response['meta']['claims_in_channel'], 0)
|
||||
|
||||
await self.stream_create('foo', '0.01', channel_id=channel_id)
|
||||
await self.stream_create('foo2', '0.01', channel_id=channel_id)
|
||||
|
||||
# resolving a channel @abc with some claims in it
|
||||
response['lbry://@abc']['confirmations'] += 2
|
||||
response['lbry://@abc']['meta']['claims_in_channel'] = 2
|
||||
response['confirmations'] += 2
|
||||
response['meta']['claims_in_channel'] = 2
|
||||
self.assertEqual(response, await self.resolve('lbry://@abc'))
|
||||
|
||||
# resolving claim foo within channel @abc
|
||||
response = await self.resolve('lbry://@abc/foo')
|
||||
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
|
||||
claim = response['lbry://@abc/foo']
|
||||
claim = await self.resolve('lbry://@abc/foo')
|
||||
self.assertEqual(claim['name'], 'foo')
|
||||
self.assertEqual(claim['value_type'], 'stream')
|
||||
self.assertEqual(claim['signing_channel']['name'], '@abc')
|
||||
@@ -60,13 +57,12 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
)
|
||||
|
||||
# resolving claim foo by itself
|
||||
self.assertEqual(claim, (await self.resolve('lbry://foo'))['lbry://foo'])
|
||||
self.assertEqual(claim, await self.resolve('lbry://foo'))
|
||||
# resolving from the given permanent url
|
||||
permanent_url = response['lbry://@abc/foo']['permanent_url']
|
||||
self.assertEqual(claim, (await self.resolve(permanent_url))[permanent_url])
|
||||
self.assertEqual(claim, await self.resolve(claim['permanent_url']))
|
||||
|
||||
# resolving multiple at once
|
||||
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(['lbry://foo', 'lbry://foo2']))
|
||||
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
|
||||
claim = response['lbry://foo2']
|
||||
self.assertEqual(claim['name'], 'foo2')
|
||||
@@ -81,7 +77,7 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
# resolve handles invalid data
|
||||
await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
|
||||
await self.generate(1)
|
||||
response = await self.resolve("lbry://gibberish")
|
||||
response = await self.out(self.daemon.jsonrpc_resolve("lbry://gibberish"))
|
||||
self.assertSetEqual({'lbry://gibberish'}, set(response))
|
||||
claim = response['lbry://gibberish']
|
||||
self.assertEqual(claim['name'], 'gibberish')
|
||||
@@ -89,10 +85,10 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
|
||||
# resolve retries
|
||||
await self.conductor.spv_node.stop()
|
||||
resolving_future = asyncio.ensure_future(self.resolve('foo'))
|
||||
resolve_task = asyncio.create_task(self.resolve('foo'))
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
await self.ledger.on_ready.first
|
||||
self.assertIsNotNone((await resolving_future)['foo']['claim_id'])
|
||||
self.assertIsNotNone((await resolve_task)['claim_id'])
|
||||
|
||||
async def test_winning_by_effective_amount(self):
|
||||
# first one remains winner unless something else changes
|
||||
@@ -182,20 +178,18 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
|
||||
# Original channel doesn't exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# only possible outside a channel
|
||||
response = await self.resolve('lbry://@abc/on-channel-claim')
|
||||
self.assertEqual(response, {
|
||||
'lbry://@abc/on-channel-claim': {
|
||||
'error': {
|
||||
'name': 'not_found',
|
||||
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
|
||||
}
|
||||
}
|
||||
})
|
||||
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
|
||||
self.assertEqual(
|
||||
{'error': {
|
||||
'name': 'NOT_FOUND',
|
||||
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
|
||||
}},
|
||||
await self.resolve('lbry://@abc/on-channel-claim')
|
||||
)
|
||||
response = await self.resolve('lbry://on-channel-claim')
|
||||
self.assertFalse(response['is_channel_signature_valid'])
|
||||
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
|
||||
response = (await self.resolve(direct_uri))[direct_uri]
|
||||
response = await self.resolve(direct_uri)
|
||||
self.assertFalse(response['is_channel_signature_valid'])
|
||||
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
|
||||
await self.stream_abandon(claim_id=orphan_claim_id)
|
||||
@@ -205,7 +199,7 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
valid_claim = await self.stream_create('on-channel-claim', '0.00000001', channel_id=channel['claim_id'])
|
||||
# resolves normally
|
||||
response = await self.resolve(uri)
|
||||
self.assertTrue(response[uri]['is_channel_signature_valid'])
|
||||
self.assertTrue(response['is_channel_signature_valid'])
|
||||
|
||||
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
|
||||
await self.stream_create(
|
||||
@@ -214,8 +208,8 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
|
||||
# it still resolves! but to the older claim
|
||||
response = await self.resolve(uri)
|
||||
self.assertTrue(response[uri]['is_channel_signature_valid'])
|
||||
self.assertEqual(response[uri]['txid'], valid_claim['txid'])
|
||||
self.assertTrue(response['is_channel_signature_valid'])
|
||||
self.assertEqual(response['txid'], valid_claim['txid'])
|
||||
claims = await self.claim_search(name='on-channel-claim')
|
||||
self.assertEqual(2, len(claims))
|
||||
self.assertEqual(
|
||||
@@ -235,8 +229,8 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
r1 = await self.resolve(f'lbry://{one}')
|
||||
r2 = await self.resolve(f'lbry://{two}')
|
||||
|
||||
self.assertEqual(winner_id, r1[f'lbry://{one}']['claim_id'])
|
||||
self.assertEqual(winner_id, r2[f'lbry://{two}']['claim_id'])
|
||||
self.assertEqual(winner_id, r1['claim_id'])
|
||||
self.assertEqual(winner_id, r2['claim_id'])
|
||||
|
||||
async def test_resolve_old_claim(self):
|
||||
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
|
||||
@@ -249,7 +243,7 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
response = await self.resolve('@olds/example')
|
||||
self.assertTrue(response['@olds/example']['is_channel_signature_valid'])
|
||||
self.assertTrue(response['is_channel_signature_valid'])
|
||||
|
||||
claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
|
||||
tx = await Transaction.claim_create(
|
||||
@@ -260,16 +254,14 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
response = await self.resolve('bad_example')
|
||||
self.assertFalse(response['bad_example']['is_channel_signature_valid'])
|
||||
response = await self.resolve('@olds/bad_example')
|
||||
self.assertEqual(response, {
|
||||
'@olds/bad_example': {
|
||||
'error': {
|
||||
'name': 'not_found',
|
||||
'text': 'Could not find claim at "@olds/bad_example".',
|
||||
}
|
||||
}
|
||||
})
|
||||
self.assertFalse(response['is_channel_signature_valid'])
|
||||
self.assertEqual(
|
||||
{'error': {
|
||||
'name': 'NOT_FOUND',
|
||||
'text': 'Could not find claim at "@olds/bad_example".',
|
||||
}},
|
||||
await self.resolve('@olds/bad_example')
|
||||
)
|
||||
|
||||
|
||||
class ResolveAfterReorg(BaseResolveTestCase):
|
||||
@@ -289,36 +281,36 @@ class ResolveAfterReorg(BaseResolveTestCase):
|
||||
channel_id = self.get_claim_id(
|
||||
await self.channel_create(channel_name, '0.01')
|
||||
)
|
||||
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
|
||||
self.assertNotIn('error', await self.resolve(channel_name))
|
||||
await self.reorg(206)
|
||||
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
|
||||
self.assertNotIn('error', await self.resolve(channel_name))
|
||||
|
||||
stream_name = 'foo'
|
||||
stream_id = self.get_claim_id(
|
||||
await self.stream_create(stream_name, '0.01', channel_id=channel_id)
|
||||
)
|
||||
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertNotIn('error', await self.resolve(stream_name))
|
||||
await self.reorg(206)
|
||||
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertNotIn('error', await self.resolve(stream_name))
|
||||
|
||||
await self.support_create(stream_id, '0.01')
|
||||
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertNotIn('error', await self.resolve(stream_name))
|
||||
await self.reorg(206)
|
||||
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertNotIn('error', await self.resolve(stream_name))
|
||||
|
||||
await self.stream_abandon(stream_id)
|
||||
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
|
||||
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertNotIn('error', await self.resolve(channel_name))
|
||||
self.assertIn('error', await self.resolve(stream_name))
|
||||
await self.reorg(206)
|
||||
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
|
||||
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertNotIn('error', await self.resolve(channel_name))
|
||||
self.assertIn('error', await self.resolve(stream_name))
|
||||
|
||||
await self.channel_abandon(channel_id)
|
||||
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
|
||||
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertIn('error', await self.resolve(channel_name))
|
||||
self.assertIn('error', await self.resolve(stream_name))
|
||||
await self.reorg(206)
|
||||
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
|
||||
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
|
||||
self.assertIn('error', await self.resolve(channel_name))
|
||||
self.assertIn('error', await self.resolve(stream_name))
|
||||
|
||||
|
||||
def generate_signed_legacy(address: bytes, output: Output):
|
||||
|
||||
@@ -2,8 +2,10 @@ import asyncio
|
||||
|
||||
import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.testcase import IntegrationTestCase
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
|
||||
|
||||
class TestSessions(IntegrationTestCase):
|
||||
@@ -49,3 +51,46 @@ class TestSegwitServer(IntegrationTestCase):
|
||||
|
||||
async def test_at_least_it_starts(self):
|
||||
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
|
||||
|
||||
|
||||
class TestUsagePayment(CommandTestCase):
|
||||
async def test_single_server_payment(self):
|
||||
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
|
||||
wallet_pay_service.payment_period = 1
|
||||
# only starts with a positive max key fee
|
||||
wallet_pay_service.max_fee = "0.0"
|
||||
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
|
||||
self.assertFalse(wallet_pay_service.running)
|
||||
wallet_pay_service.max_fee = "1.0"
|
||||
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
|
||||
self.assertTrue(wallet_pay_service.running)
|
||||
await wallet_pay_service.stop()
|
||||
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
|
||||
|
||||
address = await self.blockchain.get_raw_change_address()
|
||||
_, history = await self.ledger.get_local_status_and_history(address)
|
||||
self.assertEqual(history, [])
|
||||
|
||||
node = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.1"})
|
||||
self.addCleanup(node.stop)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.1")
|
||||
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
|
||||
|
||||
await node.stop(False)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.0")
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
|
||||
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
|
||||
self.assertEqual(tx.outputs[0].amount, 100000000)
|
||||
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
|
||||
@@ -40,7 +40,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
# And is the channel resolvable and empty?
|
||||
response = await self.resolve('lbry://@spam')
|
||||
self.assertEqual(response['lbry://@spam']['value_type'], 'channel')
|
||||
self.assertEqual(response['value_type'], 'channel')
|
||||
|
||||
# "What goes well with spam?" ponders Chris...
|
||||
# "A hovercraft with eels!" he exclaims.
|
||||
@@ -64,7 +64,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# Also checks that his new story can be found on the blockchain before
|
||||
# giving the link to all his friends.
|
||||
response = await self.resolve('lbry://@spam/hovercraft')
|
||||
self.assertEqual(response['lbry://@spam/hovercraft']['value_type'], 'stream')
|
||||
self.assertEqual(response['value_type'], 'stream')
|
||||
|
||||
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
|
||||
await self.generate(5)
|
||||
@@ -84,13 +84,12 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
await self.confirm_tx(abandon['txid'])
|
||||
|
||||
# And now checks that the claim doesn't resolve anymore.
|
||||
response = await self.resolve('lbry://@spam/hovercraft')
|
||||
self.assertEqual(
|
||||
response['lbry://@spam/hovercraft'],
|
||||
{'error': {
|
||||
'name': 'not_found',
|
||||
'name': 'NOT_FOUND',
|
||||
'text': 'Could not find claim at "lbry://@spam/hovercraft".'
|
||||
}}
|
||||
}},
|
||||
await self.resolve('lbry://@spam/hovercraft')
|
||||
)
|
||||
|
||||
# After abandoning he just waits for his LBCs to be returned to his account
|
||||
@@ -140,8 +139,8 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.resolve(uri)
|
||||
# It obviously did! Because, blockchain baby \O/
|
||||
self.assertEqual(resolve_result[uri]['amount'], '1.0')
|
||||
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.2')
|
||||
self.assertEqual(resolve_result['amount'], '1.0')
|
||||
self.assertEqual(resolve_result['meta']['effective_amount'], '1.2')
|
||||
await self.generate(5)
|
||||
|
||||
# Now he also wanted to support the original creator of the Award Winning Novel
|
||||
@@ -154,7 +153,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# And again checks if it went to the just right place
|
||||
resolve_result = await self.resolve(uri)
|
||||
# Which it obviously did. Because....?????
|
||||
self.assertEqual(resolve_result[uri]['meta']['effective_amount'], '1.5')
|
||||
self.assertEqual(resolve_result['meta']['effective_amount'], '1.5')
|
||||
await self.generate(5)
|
||||
|
||||
# Seeing the ravishing success of his novel Chris adds support to his claim too
|
||||
@@ -188,11 +187,10 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
await self.confirm_tx(abandon['txid'])
|
||||
|
||||
# He them checks that the claim doesn't resolve anymore.
|
||||
response = await self.resolve(uri)
|
||||
self.assertEqual(
|
||||
response[uri],
|
||||
{'error': {
|
||||
'name': 'not_found',
|
||||
'name': 'NOT_FOUND',
|
||||
'text': f'Could not find claim at "{uri}".'
|
||||
}}
|
||||
}},
|
||||
await self.resolve(uri)
|
||||
)
|
||||
@@ -7,7 +7,7 @@ from lbry.extras import cli
|
||||
from lbry.extras.daemon.components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
|
||||
@@ -22,7 +22,7 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
Daemon.component_attributes = {}
|
||||
self.daemon = Daemon(conf)
|
||||
|
||||
@@ -278,7 +278,9 @@ class CommentCommands(CommandTestCase):
|
||||
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
|
||||
stream = (await self.stream_create())['outputs'][0]
|
||||
|
||||
self.assertEqual(0, len((await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']))
|
||||
empty_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
|
||||
self.assertEqual(0, len(empty_list['items']))
|
||||
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=stream['claim_id'],
|
||||
channel_id=channel['claim_id'],
|
||||
@@ -296,66 +298,41 @@ class CommentCommands(CommandTestCase):
|
||||
comment='Let\'s all go to Margaritaville',
|
||||
parent_id=comments[0]['comment_id']
|
||||
)
|
||||
|
||||
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
|
||||
self.assertEqual(2, len(comments))
|
||||
self.assertEqual(comments[0]['channel_id'], channel2['claim_id'])
|
||||
self.assertEqual(comments[0]['parent_id'], comments[1]['comment_id'])
|
||||
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=stream['claim_id'],
|
||||
comment='Anonymous comment'
|
||||
)
|
||||
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
|
||||
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
|
||||
|
||||
async def test02_unsigned_comment_list(self):
|
||||
stream = (await self.stream_create())['outputs'][0]
|
||||
comments = []
|
||||
num_items = 28
|
||||
for i in range(num_items):
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment=f'{i}',
|
||||
claim_id=stream['claim_id'],
|
||||
)
|
||||
self.assertIn('comment_id', comment)
|
||||
comments.append(comment)
|
||||
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
|
||||
comment_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
|
||||
for field in list_fields:
|
||||
self.assertIn(field, comment_list)
|
||||
self.assertEqual(comment_list['total_items'], num_items)
|
||||
for comment in comment_list['items']:
|
||||
self.assertEqual(comment['comment'], comments.pop()['comment'])
|
||||
|
||||
signed_comment_list = await self.daemon.jsonrpc_comment_list(
|
||||
claim_id=stream['claim_id'],
|
||||
is_channel_signature_valid=True
|
||||
)
|
||||
self.assertIs(len(signed_comment_list['items']), 0)
|
||||
|
||||
async def test03_signed_comments_list(self):
|
||||
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
|
||||
stream = (await self.stream_create())['outputs'][0]
|
||||
comments = []
|
||||
|
||||
for i in range(28):
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment=f'{i}',
|
||||
claim_id=stream['claim_id'],
|
||||
channel_id=channel['claim_id'],
|
||||
)
|
||||
self.assertIn('comment_id', comment)
|
||||
comments.append(comment)
|
||||
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
|
||||
self.assertIn('comment_id', comment)
|
||||
|
||||
comment_list = await self.daemon.jsonrpc_comment_list(
|
||||
claim_id=stream['claim_id']
|
||||
)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, comment_list)
|
||||
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
set(comment_list)
|
||||
)
|
||||
|
||||
self.assertIs(comment_list['page_size'], 50)
|
||||
self.assertIs(comment_list['page'], 1)
|
||||
self.assertIs(comment_list['total_items'], 28)
|
||||
for comment in comment_list['items']:
|
||||
self.assertEqual(comment['comment'], comments.pop()['comment'])
|
||||
comment_temp = comments.pop()
|
||||
self.assertEqual(comment['comment'], comment_temp['comment'])
|
||||
|
||||
signed_comment_list = await self.daemon.jsonrpc_comment_list(
|
||||
claim_id=stream['claim_id'],
|
||||
@@ -365,13 +342,15 @@ class CommentCommands(CommandTestCase):
|
||||
|
||||
async def test04_comment_abandons(self):
|
||||
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
|
||||
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
|
||||
stream = (await self.stream_create('Pawnee_Tow_Hall_of_Fame_by_Leslie_Knope'))['outputs'][0]
|
||||
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
|
||||
claim_id=stream['claim_id'],
|
||||
channel_id=rswanson['claim_id']
|
||||
)
|
||||
self.assertIn('signature', comment)
|
||||
|
||||
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
|
||||
self.assertIn(comment['comment_id'], abandoned)
|
||||
self.assertTrue(abandoned[comment['comment_id']]['abandoned'])
|
||||
@@ -383,7 +362,7 @@ class CommentCommands(CommandTestCase):
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
moth_id = moth['claim_id']
|
||||
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
|
||||
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
|
||||
claim_id = stream['outputs'][0]['claim_id']
|
||||
|
||||
comment1 = await self.daemon.jsonrpc_comment_create(
|
||||
@@ -436,12 +415,12 @@ class CommentCommands(CommandTestCase):
|
||||
for item in items_visible + items_hidden:
|
||||
self.assertIn(item, comments['items'])
|
||||
|
||||
async def test06_comment_list_test(self):
|
||||
async def test06_comment_list(self):
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
moth_id = moth['claim_id']
|
||||
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
|
||||
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
|
||||
claim_id = stream['outputs'][0]['claim_id']
|
||||
|
||||
hidden_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Who on earth would want to sit around on a lamp all day',
|
||||
claim_id=claim_id,
|
||||
@@ -451,44 +430,48 @@ class CommentCommands(CommandTestCase):
|
||||
owner_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Go away you yellow freak',
|
||||
claim_id=claim_id,
|
||||
channel_id=moth_id,
|
||||
channel_id=moth['claim_id'],
|
||||
)
|
||||
other_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='I got my swim trunks and my flippy-floppies',
|
||||
claim_id=claim_id,
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
anon_comment = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=claim_id,
|
||||
comment='Anonymous comment'
|
||||
)
|
||||
all_comments = [anon_comment, other_comment, owner_comment, hidden_comment]
|
||||
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
|
||||
all_comments = [other_comment, owner_comment, hidden_comment]
|
||||
|
||||
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, normal_list)
|
||||
self.assertEqual(normal_list['total_items'], 4)
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
set(normal_list)
|
||||
)
|
||||
self.assertEqual(normal_list['total_items'], 3)
|
||||
self.assertTrue(normal_list['has_hidden_comments'])
|
||||
for i, cmnt in enumerate(all_comments):
|
||||
self.assertEqual(cmnt['comment_id'], normal_list['items'][i]['comment_id'])
|
||||
|
||||
hidden = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
set(hidden)
|
||||
)
|
||||
self.assertTrue(hidden['has_hidden_comments'])
|
||||
for field in list_fields:
|
||||
self.assertIn(field, hidden)
|
||||
self.assertEqual(hidden['total_items'], 1)
|
||||
|
||||
visible = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, visible)
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
set(visible)
|
||||
)
|
||||
self.assertTrue(visible['has_hidden_comments'])
|
||||
self.assertEqual(visible['total_items'], normal_list['total_items'] - hidden['total_items'])
|
||||
|
||||
valid_list = await self.daemon.jsonrpc_comment_list(claim_id, is_channel_signature_valid=True)
|
||||
for field in list_fields:
|
||||
self.assertIn(field, valid_list)
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
set(valid_list)
|
||||
)
|
||||
self.assertTrue(visible['has_hidden_comments'])
|
||||
self.assertEqual(len(valid_list['items']), len(normal_list['items']) - 1)
|
||||
self.assertEqual(len(valid_list['items']), len(normal_list['items']))
|
||||
|
||||
async def test07_edit_comments(self):
|
||||
luda = (await self.channel_create('@Ludacris'))['outputs'][0]
|
||||
@@ -537,18 +520,3 @@ class CommentCommands(CommandTestCase):
|
||||
comment='If you see it and you mean then you know you have to go',
|
||||
comment_id=original_cid
|
||||
)
|
||||
|
||||
# editing an anonymous comment
|
||||
anon_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='fast and furiouuuuuus',
|
||||
claim_id=claim_id
|
||||
)
|
||||
|
||||
anon_cid = anon_comment.get('comment_id')
|
||||
self.assertIsNotNone(anon_cid)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
await self.daemon.jsonrpc_comment_update(
|
||||
comment='drift drift drift',
|
||||
comment_id=anon_cid
|
||||
)
|
||||
@@ -1,3 +1,4 @@
|
||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
@@ -11,6 +12,7 @@ class AddressManagement(CommandTestCase):
|
||||
self.assertItemCount(single, 1)
|
||||
self.assertEqual(single['items'][0], addresses['items'][11])
|
||||
|
||||
|
||||
class SettingsManagement(CommandTestCase):
|
||||
|
||||
async def test_settings(self):
|
||||
@@ -21,5 +23,18 @@ class SettingsManagement(CommandTestCase):
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001))
|
||||
|
||||
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('spv1.lbry.com', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv1.lbry.com', 50001))
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001))
|
||||
|
||||
# test_privacy_settings (merged for reducing test time, unmerge when its fast)
|
||||
# tests that changing share_usage_data propagates to the relevant properties
|
||||
self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data'])
|
||||
loggly = get_loggly_handler(self.daemon.conf)
|
||||
self.addCleanup(loggly.close)
|
||||
self.assertFalse(self.daemon.analytics_manager.enabled)
|
||||
self.assertFalse(loggly.enabled)
|
||||
self.daemon.jsonrpc_settings_set('share_usage_data', True)
|
||||
self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data'])
|
||||
self.assertTrue(self.daemon.analytics_manager.enabled)
|
||||
self.assertTrue(loggly.enabled)
|
||||
self.daemon.jsonrpc_settings_set('share_usage_data', False)
|
||||
+8
-3
@@ -2,7 +2,7 @@ import logging
|
||||
import pathlib
|
||||
import time
|
||||
|
||||
from .test_claim_commands import ClaimTestCase
|
||||
from ..blockchain.test_claim_commands import ClaimTestCase
|
||||
from lbry.conf import TranscodeConfig
|
||||
from lbry.file_analysis import VideoFileAnalyzer
|
||||
|
||||
@@ -93,6 +93,11 @@ class TranscodeValidation(ClaimTestCase):
|
||||
fixed_file = await self.analyzer.verify_or_repair(True, True, file_name)
|
||||
pathlib.Path(fixed_file).unlink()
|
||||
|
||||
async def test_max_bit_rate(self):
|
||||
self.conf.video_bitrate_maximum = 100
|
||||
with self.assertRaisesRegex(Exception, "The bit rate is above the configured maximum"):
|
||||
await self.analyzer.verify_or_repair(True, False, self.video_file_name)
|
||||
|
||||
async def test_video_format(self):
|
||||
file_name = self.make_name("bad_video_format_1")
|
||||
if not file_name.exists():
|
||||
@@ -130,7 +135,7 @@ class TranscodeValidation(ClaimTestCase):
|
||||
|
||||
scan_data = await self.analyzer._get_scan_data(True, self.video_file_ogg)
|
||||
extension = self.analyzer._get_best_container_extension(scan_data, "")
|
||||
self.assertEqual(extension, "ogg")
|
||||
self.assertEqual(extension, "ogv")
|
||||
|
||||
scan_data = await self.analyzer._get_scan_data(True, self.video_file_webm)
|
||||
extension = self.analyzer._get_best_container_extension(scan_data, "")
|
||||
@@ -143,7 +148,7 @@ class TranscodeValidation(ClaimTestCase):
|
||||
self.assertEqual("webm", extension)
|
||||
|
||||
extension = self.analyzer._get_best_container_extension("", "libtheora")
|
||||
self.assertEqual("ogg", extension)
|
||||
self.assertEqual("ogv", extension)
|
||||
|
||||
async def test_no_ffmpeg(self):
|
||||
self.conf.ffmpeg_folder = "I don't really exist/"
|
||||
@@ -27,6 +27,7 @@ class TestComponentManager(AsyncioTestCase):
|
||||
components.HashAnnouncerComponent,
|
||||
components.PeerProtocolServerComponent,
|
||||
components.StreamManagerComponent,
|
||||
components.WalletServerPaymentsComponent
|
||||
]
|
||||
]
|
||||
self.component_manager = ComponentManager(Config())
|
||||
|
||||
@@ -57,10 +57,6 @@ class TestURLParsing(unittest.TestCase):
|
||||
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
|
||||
# unicode regex edges
|
||||
_url = lambda name: url(name, stream_name=name)
|
||||
_url('\u0009')
|
||||
_url('\u000a')
|
||||
_url('\u000d')
|
||||
_url('\u0020')
|
||||
_url('\uD799')
|
||||
_url('\uE000')
|
||||
_url('\uFFFD')
|
||||
@@ -79,6 +75,10 @@ class TestURLParsing(unittest.TestCase):
|
||||
fail("lbry://\uDFFE")
|
||||
fail("lbry://\uFFFF")
|
||||
fail("lbry://;")
|
||||
fail("lbry://no\ttab")
|
||||
fail("lbry://no space")
|
||||
fail("lbry://no\rcr")
|
||||
fail("lbry://no\new\nline")
|
||||
fail("lbry://\"")
|
||||
fail("lbry://\\")
|
||||
fail("lbry:///")
|
||||
|
||||
@@ -122,6 +122,10 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
|
||||
|
||||
class TestStreamManager(BlobExchangeTestBase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.client_config.share_usage_data = True
|
||||
|
||||
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
|
||||
file_path = os.path.join(self.server_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
|
||||
+12
-6
@@ -64,8 +64,9 @@ class CLILoggingTest(AsyncioTestCase):
|
||||
self.assertTrue(log.isEnabledFor(logging.INFO))
|
||||
self.assertFalse(log.isEnabledFor(logging.DEBUG))
|
||||
self.assertFalse(log.isEnabledFor(logging.DEBUG))
|
||||
self.assertEqual(len(log.handlers), 1)
|
||||
self.assertEqual(len(log.handlers), 2)
|
||||
self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler)
|
||||
self.assertFalse(log.handlers[1].enabled)
|
||||
|
||||
async with get_logger(["start", "--verbose"]) as log:
|
||||
self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG))
|
||||
@@ -80,16 +81,22 @@ class CLILoggingTest(AsyncioTestCase):
|
||||
self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG))
|
||||
|
||||
async def test_loggly(self):
|
||||
async with get_logger(["start"], share_usage_data=False) as log:
|
||||
self.assertEqual(len(log.getChild("lbry").handlers), 2) # file and console
|
||||
async with get_logger(["start"]) as log: # default share_usage_data=True
|
||||
async with get_logger(["start"]) as log: # default share_usage_data=False
|
||||
log = log.getChild("lbry")
|
||||
self.assertIsInstance(log.handlers[0], logging.StreamHandler)
|
||||
self.assertIsInstance(log.handlers[1], logging.StreamHandler)
|
||||
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
|
||||
self.assertFalse(log.handlers[2].enabled)
|
||||
async with get_logger(["start"], share_usage_data=True) as log:
|
||||
log = log.getChild("lbry")
|
||||
self.assertEqual(len(log.handlers), 3)
|
||||
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
|
||||
async with get_logger(["start"], share_usage_data=True) as log: # explicit share_usage_data=True
|
||||
self.assertTrue(log.handlers[2].enabled)
|
||||
async with get_logger(["start"], share_usage_data=False) as log: # explicit share_usage_data=False
|
||||
log = log.getChild("lbry")
|
||||
self.assertEqual(len(log.handlers), 3)
|
||||
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
|
||||
self.assertFalse(log.handlers[2].enabled)
|
||||
|
||||
async def test_quiet(self):
|
||||
async with get_logger(["start"]) as log: # default is loud
|
||||
@@ -100,7 +107,6 @@ class CLILoggingTest(AsyncioTestCase):
|
||||
log = log.getChild("lbry")
|
||||
self.assertEqual(len(log.handlers), 2)
|
||||
self.assertIsNot(type(log.handlers[0]), logging.StreamHandler)
|
||||
self.assertIsNot(type(log.handlers[1]), logging.StreamHandler)
|
||||
|
||||
|
||||
class CLITest(AsyncioTestCase):
|
||||
|
||||
@@ -98,6 +98,12 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
'txo_name__in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.name__in': {'abc123'}}),
|
||||
('txo.name = :txo_name__in0', {
|
||||
'txo_name__in0': 'abc123',
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
|
||||
('txo.age IN (SELECT age from ages_table)', {})
|
||||
@@ -118,6 +124,12 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
'txo_name__not_in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.name__not_in': ('abc123',)}),
|
||||
('txo.name != :txo_name__not_in0', {
|
||||
'txo_name__not_in0': 'abc123',
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
|
||||
('txo.age NOT IN (SELECT age from ages_table)', {})
|
||||
|
||||
Reference in new issue
Block a user