Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7582c221d1 | ||
|
|
c109895848 | ||
|
|
eccedada40 | ||
|
|
25d54accf8 | ||
|
|
d07685f0e9 | ||
|
|
2445c00c7e | ||
|
|
4c1d3ef514 | ||
|
|
4614c7d4c2 | ||
|
|
bbf1ef0dc3 | ||
|
|
3433c9e708 | ||
|
|
2cd5d75a2e | ||
|
|
2535b8adef | ||
|
|
4edab7bb7f | ||
|
|
fd8658e317 | ||
|
|
51d21d8c86 | ||
|
|
b4c3307cdf | ||
|
|
4e8d10cb44 | ||
|
|
e96875a425 | ||
|
|
5ab0035348 | ||
|
|
4ddff96b1e | ||
|
|
a08d84c1df | ||
|
|
21c71bfac1 | ||
|
|
6baaed3581 | ||
|
|
152dbfd5d1 | ||
|
|
a56d14086b | ||
|
|
aee87693f8 | ||
|
|
976b4affd9 | ||
|
|
e222b6ad9c | ||
|
|
19b17374e8 | ||
|
|
43989122bb | ||
|
|
72712d6047 | ||
|
|
0b52d2cc15 | ||
|
|
8304102136 | ||
|
|
3381aefcfa | ||
|
|
279a365cb1 | ||
|
|
2c9e00da56 |
No files matched your search
@@ -31,8 +31,8 @@ jobs:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
#- macos-latest
|
||||
#- windows-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -49,10 +49,18 @@ jobs:
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,test]
|
||||
- env:
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
run: pip install -e .[torrent,test]
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: make test-unit-coverage
|
||||
- if: startsWith(runner.os, 'linux') != true
|
||||
run: pip install -e .[test]
|
||||
- if: startsWith(runner.os, 'linux') != true
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: python -m unittest tests/unit/test_conf.py
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
|
||||
@@ -15,8 +15,8 @@ jobs:
|
||||
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||
- uses: slackapi/slack-github-action@v1.14.0
|
||||
env:
|
||||
CHANGELOG: ${{ toJSON(steps.markdown.outputs.text) }}
|
||||
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||
with:
|
||||
payload: '{"type": "mrkdwn", "text": ${{ env.CHANGELOG }}}'
|
||||
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }}'
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.101.1"
|
||||
__version__ = "0.102.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -0,0 +1,64 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DiskSpaceManager:
|
||||
|
||||
def __init__(self, config, cleaning_interval=30 * 60):
|
||||
self.config = config
|
||||
self.cleaning_interval = cleaning_interval
|
||||
self.running = False
|
||||
self.task = None
|
||||
|
||||
@property
|
||||
def space_used_bytes(self):
|
||||
used = 0
|
||||
data_dir = os.path.join(self.config.data_dir, 'blobfiles')
|
||||
for item in os.scandir(data_dir):
|
||||
if item.is_file:
|
||||
used += item.stat().st_size
|
||||
return used
|
||||
|
||||
@property
|
||||
def space_used_mb(self):
|
||||
return int(self.space_used_bytes/1024.0/1024.0)
|
||||
|
||||
def clean(self):
|
||||
if not self.config.blob_storage_limit:
|
||||
return 0
|
||||
used = 0
|
||||
files = []
|
||||
data_dir = os.path.join(self.config.data_dir, 'blobfiles')
|
||||
for file in os.scandir(data_dir):
|
||||
if file.is_file:
|
||||
file_stats = file.stat()
|
||||
used += file_stats.st_size
|
||||
files.append((file_stats.st_mtime, file_stats.st_size, file.path))
|
||||
files.sort()
|
||||
available = (self.config.blob_storage_limit*1024*1024) - used
|
||||
cleaned = 0
|
||||
for _, file_size, file in files:
|
||||
available += file_size
|
||||
if available > 0:
|
||||
break
|
||||
os.remove(file)
|
||||
cleaned += 1
|
||||
return cleaned
|
||||
|
||||
async def cleaning_loop(self):
|
||||
while self.running:
|
||||
await asyncio.sleep(self.cleaning_interval)
|
||||
await asyncio.get_event_loop().run_in_executor(None, self.clean)
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self.task = asyncio.create_task(self.cleaning_loop())
|
||||
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self.task.cancel()
|
||||
@@ -634,6 +634,7 @@ class Config(CLIConfig):
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
|
||||
+2
-2
@@ -156,9 +156,9 @@ class KademliaPeer:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
|
||||
@@ -452,7 +452,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
except ValueError as err:
|
||||
log.warning("error replying to %s: %s", address[0], str(err))
|
||||
return
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
|
||||
@@ -53,13 +53,18 @@ Code | Name | Message
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
423 | TooManyClaimSearchParameters | {key} cant be set for more than {limit} items.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
|
||||
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
|
||||
436 | WalletNotFound | Wallet not found at {wallet_path}.
|
||||
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
@@ -242,6 +242,14 @@ class IncompatibleWalletServerError(WalletError):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class TooManyClaimSearchParametersError(WalletError):
|
||||
|
||||
def __init__(self, key, limit):
|
||||
self.key = key
|
||||
self.limit = limit
|
||||
super().__init__(f"{key} cant be set for more than {limit} items.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
@@ -263,6 +271,34 @@ class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class WalletNotLoadedError(WalletError):
|
||||
|
||||
def __init__(self, wallet_id):
|
||||
self.wallet_id = wallet_id
|
||||
super().__init__(f"Wallet {wallet_id} is not loaded.")
|
||||
|
||||
|
||||
class WalletAlreadyLoadedError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet {wallet_path} is already loaded.")
|
||||
|
||||
|
||||
class WalletNotFoundError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet not found at {wallet_path}.")
|
||||
|
||||
|
||||
class WalletAlreadyExistsError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
|
||||
@@ -15,6 +15,7 @@ from lbry.dht.node import Node
|
||||
from lbry.dht.peer import is_valid_public_ipv4
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob.disk_space_manager import DiskSpaceManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
@@ -40,6 +41,7 @@ WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
DISK_SPACE_COMPONENT = "disk_space"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
@@ -375,6 +377,30 @@ class FileManagerComponent(Component):
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class DiskSpaceComponent(Component):
|
||||
component_name = DISK_SPACE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.disk_space_manager = DiskSpaceManager(self.conf)
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[DiskSpaceManager]:
|
||||
return self.disk_space_manager
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'space_used': str(self.disk_space_manager.space_used_mb),
|
||||
'running': self.disk_space_manager.running,
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
await self.disk_space_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
await self.disk_space_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
|
||||
@@ -36,12 +36,12 @@ from lbry.blob_exchange.downloader import download_blob
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import (
|
||||
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
|
||||
CommandDoesNotExistError
|
||||
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError
|
||||
)
|
||||
from lbry.extras import system_info
|
||||
from lbry.extras.daemon import analytics
|
||||
from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT
|
||||
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT
|
||||
from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT
|
||||
from lbry.extras.daemon.componentmanager import RequiredCondition
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
@@ -56,7 +56,7 @@ from lbry.schema.url import URL
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.dht.node import Node
|
||||
from lbry.extras.daemon.components import UPnPComponent
|
||||
from lbry.extras.daemon.components import UPnPComponent, DiskSpaceManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Ledger
|
||||
@@ -375,6 +375,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
def blob_manager(self) -> typing.Optional['BlobManager']:
|
||||
return self.component_manager.get_component(BLOB_COMPONENT)
|
||||
|
||||
@property
|
||||
def disk_space_manager(self) -> typing.Optional['DiskSpaceManager']:
|
||||
return self.component_manager.get_component(DISK_SPACE_COMPONENT)
|
||||
|
||||
@property
|
||||
def upnp(self) -> typing.Optional['UPnPComponent']:
|
||||
return self.component_manager.get_component(UPNP_COMPONENT)
|
||||
@@ -698,7 +702,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
raise
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
self.failed_request_metric.labels(method=function_name).inc()
|
||||
log.exception("error handling api request")
|
||||
if not isinstance(e, BaseError):
|
||||
log.exception("error handling api request")
|
||||
else:
|
||||
log.error("error handling api request: %s", e)
|
||||
return JSONRPCError.create_command_exception(
|
||||
command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc()
|
||||
)
|
||||
@@ -1103,6 +1110,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not stream:
|
||||
raise DownloadSDTimeoutError(uri)
|
||||
except Exception as e:
|
||||
# TODO: use error from lbry.error
|
||||
log.warning("Error downloading %s: %s", uri, str(e))
|
||||
return {"error": str(e)}
|
||||
return stream
|
||||
@@ -1269,9 +1277,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id)
|
||||
for wallet in self.wallet_manager.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
raise Exception(f"Wallet at path '{wallet_path}' already exists and is loaded.")
|
||||
raise WalletAlreadyLoadedError(wallet_path)
|
||||
if os.path.exists(wallet_path):
|
||||
raise Exception(f"Wallet at path '{wallet_path}' already exists, use 'wallet_add' to load wallet.")
|
||||
raise WalletAlreadyExistsError(wallet_path)
|
||||
|
||||
wallet = self.wallet_manager.import_wallet(wallet_path)
|
||||
if not wallet.accounts and create_account:
|
||||
@@ -1304,9 +1312,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id)
|
||||
for wallet in self.wallet_manager.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
raise Exception(f"Wallet at path '{wallet_path}' is already loaded.")
|
||||
raise WalletAlreadyLoadedError(wallet_path)
|
||||
if not os.path.exists(wallet_path):
|
||||
raise Exception(f"Wallet at path '{wallet_path}' was not found.")
|
||||
raise WalletNotFoundError(wallet_path)
|
||||
wallet = self.wallet_manager.import_wallet(wallet_path)
|
||||
if self.ledger.network.is_connected:
|
||||
for account in wallet.accounts:
|
||||
@@ -1491,7 +1499,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported address: '{address}'")
|
||||
raise ValueError(f"Unsupported address: '{address}'") # TODO: use error from lbry.error
|
||||
|
||||
tx = await Transaction.create(
|
||||
[], outputs, accounts, account
|
||||
@@ -1769,8 +1777,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
from_account = wallet.get_account_or_default(from_account)
|
||||
amount = self.get_dewies_or_error('amount', amount) if amount else None
|
||||
if not isinstance(outputs, int):
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError("--outputs must be an integer.")
|
||||
if everything and outputs > 1:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError("Using --everything along with --outputs is not supported.")
|
||||
return from_account.fund(
|
||||
to_account=to_account, amount=amount, everything=everything,
|
||||
@@ -2041,10 +2051,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
if status not in ['start', 'stop']:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Status must be "start" or "stop".')
|
||||
|
||||
streams = self.file_manager.get_filtered(**kwargs)
|
||||
if not streams:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f'Unable to find a file for {kwargs}')
|
||||
stream = streams[0]
|
||||
if status == 'start' and not stream.running:
|
||||
@@ -2227,20 +2239,25 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if claim_id:
|
||||
txo = await self.ledger.get_claim_by_claim_id(accounts, claim_id, include_purchase_receipt=True)
|
||||
if not isinstance(txo, Output) or not txo.is_claim:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Could not find claim with claim_id '{claim_id}'. ")
|
||||
elif url:
|
||||
txo = (await self.ledger.resolve(accounts, [url], include_purchase_receipt=True))[url]
|
||||
if not isinstance(txo, Output) or not txo.is_claim:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Could not find claim with url '{url}'. ")
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Missing argument claim_id or url. ")
|
||||
if not allow_duplicate_purchase and txo.purchase_receipt:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"You already have a purchase for claim_id '{claim_id}'. "
|
||||
f"Use --allow-duplicate-purchase flag to override."
|
||||
)
|
||||
claim = txo.claim
|
||||
if not claim.is_stream or not claim.stream.has_fee:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Claim '{claim_id}' does not have a purchase price.")
|
||||
tx = await self.wallet_manager.create_purchase_transaction(
|
||||
accounts, txo, self.exchange_rate_manager, override_max_key_fee
|
||||
@@ -2482,6 +2499,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if "claim_ids" in kwargs and not kwargs["claim_ids"]:
|
||||
kwargs.pop("claim_ids")
|
||||
if {'claim_id', 'claim_ids'}.issubset(kwargs):
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
|
||||
if kwargs.pop('valid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 1
|
||||
@@ -2601,6 +2619,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
existing_channels = await self.ledger.get_channels(accounts=wallet.accounts, claim_name=name)
|
||||
if len(existing_channels) > 0:
|
||||
if not allow_duplicate_name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"You already have a channel under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
@@ -2733,11 +2752,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(existing_channels) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"Can't find the channel '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
old_txo = existing_channels[0]
|
||||
if not old_txo.claim.is_channel:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a channel."
|
||||
)
|
||||
@@ -2860,9 +2881,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet=wallet, accounts=accounts, claim_id=claim_id
|
||||
)
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Must specify claim_id, or txid and nout')
|
||||
|
||||
if not claims:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('No claim found for the specified claim_id or txid:nout')
|
||||
|
||||
tx = await Transaction.create(
|
||||
@@ -2931,6 +2954,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
address = channel.get_address(self.ledger)
|
||||
public_key = await self.ledger.get_public_key_for_address(wallet, address)
|
||||
if not public_key:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Can't find public key for address holding the channel.")
|
||||
export = {
|
||||
'name': channel.claim_name,
|
||||
@@ -2992,6 +3016,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await self.ledger._update_tasks.done.wait()
|
||||
# Case 3: the holding address has changed and we can't create or find an account for it
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
"Channel owning account has changed since the channel was exported and "
|
||||
"it is not an account to which you have access."
|
||||
@@ -3117,11 +3142,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(claims) == 0:
|
||||
if 'bid' not in kwargs:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("'bid' is a required argument for new publishes.")
|
||||
return await self.jsonrpc_stream_create(name, **kwargs)
|
||||
elif len(claims) == 1:
|
||||
assert claims[0].claim.is_stream, f"Claim at name '{name}' is not a stream claim."
|
||||
return await self.jsonrpc_stream_update(claims[0].claim_id, replace=True, **kwargs)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"There are {len(claims)} claims for '{name}', please use 'stream update' command "
|
||||
f"to update a specific stream claim."
|
||||
@@ -3173,11 +3200,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
claims = await account.get_claims(claim_name=name)
|
||||
if len(claims) > 0:
|
||||
if not allow_duplicate_name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"You already have a stream claim published under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
if not VALID_FULL_CLAIM_ID.fullmatch(claim_id):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Invalid claim id. It is expected to be a 40 characters long hexadecimal string.')
|
||||
|
||||
claim = Claim()
|
||||
@@ -3321,6 +3350,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
claims = await account.get_claims(claim_name=name)
|
||||
if len(claims) > 0:
|
||||
if not allow_duplicate_name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"You already have a stream claim published under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
@@ -3504,11 +3534,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(existing_claims) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"Can't find the stream '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
old_txo = existing_claims[0]
|
||||
if not old_txo.claim.is_stream:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream claim."
|
||||
)
|
||||
@@ -3636,9 +3668,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet=wallet, accounts=accounts, claim_id=claim_id
|
||||
)
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Must specify claim_id, or txid and nout')
|
||||
|
||||
if not claims:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('No claim found for the specified claim_id or txid:nout')
|
||||
|
||||
tx = await Transaction.create(
|
||||
@@ -3800,6 +3834,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
existing_collections = await self.ledger.get_collections(accounts=wallet.accounts, claim_name=name)
|
||||
if len(existing_collections) > 0:
|
||||
if not allow_duplicate_name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"You already have a collection under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
@@ -3923,11 +3958,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(existing_collections) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"Can't find the collection '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
old_txo = existing_collections[0]
|
||||
if not old_txo.claim.is_collection:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a collection."
|
||||
)
|
||||
@@ -4053,12 +4090,15 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if claim_id:
|
||||
txo = await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id)
|
||||
if not isinstance(txo, Output) or not txo.is_claim:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Could not find collection with claim_id '{claim_id}'. ")
|
||||
elif url:
|
||||
txo = (await self.ledger.resolve(wallet.accounts, [url]))[url]
|
||||
if not isinstance(txo, Output) or not txo.is_claim:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Could not find collection with url '{url}'. ")
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Missing argument claim_id or url. ")
|
||||
|
||||
page_num, page_size = abs(page), min(abs(page_size), 50)
|
||||
@@ -4233,9 +4273,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
wallet=wallet, accounts=accounts, claim_id=claim_id
|
||||
)
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Must specify claim_id, or txid and nout')
|
||||
|
||||
if not supports:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('No supports found for the specified claim_id or txid:nout')
|
||||
|
||||
if keep is not None:
|
||||
@@ -4470,6 +4512,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
elif order_by in ('height', 'amount', 'none'):
|
||||
constraints['order_by'] = order_by
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError(f"'{order_by}' is not a valid --order_by value.")
|
||||
self._constrain_txo_from_kwargs(constraints, **kwargs)
|
||||
return paginate_rows(claims, None if no_totals else claim_count, page, page_size, **constraints)
|
||||
@@ -4776,10 +4819,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("invalid blob hash")
|
||||
if search_bottom_out_limit is not None:
|
||||
search_bottom_out_limit = int(search_bottom_out_limit)
|
||||
if search_bottom_out_limit <= 0:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("invalid bottom out limit")
|
||||
else:
|
||||
search_bottom_out_limit = 4
|
||||
@@ -4823,12 +4868,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
blob_hashes.append(blob_hash)
|
||||
elif stream_hash or sd_hash:
|
||||
if sd_hash and stream_hash:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("either the sd hash or the stream hash should be provided, not both")
|
||||
if sd_hash:
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
blobs = await self.storage.get_blobs_for_stream(stream_hash, only_completed=True)
|
||||
blob_hashes.extend(blob.blob_hash for blob in blobs if blob.blob_hash is not None)
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('single argument must be specified')
|
||||
await self.storage.should_single_announce_blobs(blob_hashes, immediate=True)
|
||||
return True
|
||||
@@ -4916,6 +4963,22 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
raise NotImplementedError()
|
||||
|
||||
@requires(DISK_SPACE_COMPONENT)
|
||||
async def jsonrpc_blob_clean(self):
|
||||
"""
|
||||
Deletes blobs to cleanup disk space
|
||||
|
||||
Usage:
|
||||
blob_clean
|
||||
|
||||
Options:
|
||||
None
|
||||
|
||||
Returns:
|
||||
(bool) true if successful
|
||||
"""
|
||||
return self.disk_space_manager.clean()
|
||||
|
||||
@requires(FILE_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_reflect(self, **kwargs):
|
||||
"""
|
||||
@@ -5079,6 +5142,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
if not tracemalloc.is_tracing():
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Enable tracemalloc first! See 'tracemalloc set' command.")
|
||||
stats = tracemalloc.take_snapshot().filter_traces((
|
||||
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
|
||||
@@ -5288,6 +5352,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
comment_id=comment_id
|
||||
)
|
||||
if 'error' in channel:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError(channel['error'])
|
||||
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
@@ -5544,51 +5609,64 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
allow_script_address and self.ledger.is_script_address(address)
|
||||
)
|
||||
except:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"'{address}' is not a valid address")
|
||||
|
||||
@staticmethod
|
||||
def valid_stream_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Stream name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
"Stream names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Stream name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Invalid stream name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_collection_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Collection name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception('Collection name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Invalid collection name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_channel_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
"Channel name cannot be blank."
|
||||
)
|
||||
parsed = URL.parse(name)
|
||||
if not parsed.has_channel:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Channel names must start with '@' symbol.")
|
||||
if parsed.channel.name != name:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Channel name has invalid character")
|
||||
except (TypeError, ValueError):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("Invalid channel name.")
|
||||
|
||||
def get_fee_address(self, kwargs: dict, claim_address: str) -> str:
|
||||
@@ -5620,6 +5698,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
elif channel_name:
|
||||
key, value = 'name', channel_name
|
||||
else:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError("Couldn't find channel because a channel_id or channel_name was not provided.")
|
||||
channels = await self.ledger.get_channels(
|
||||
wallet=wallet, accounts=wallet.get_accounts_or_all(account_ids),
|
||||
@@ -5627,13 +5706,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(channels) == 1:
|
||||
if for_signing and not channels[0].has_private_key:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
|
||||
return channels[0]
|
||||
elif len(channels) > 1:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError(
|
||||
f"Multiple channels found with channel_{key} '{value}', "
|
||||
f"pass a channel_id to narrow it down."
|
||||
)
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError(f"Couldn't find channel with channel_{key} '{value}'.")
|
||||
|
||||
@staticmethod
|
||||
@@ -5641,9 +5723,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
try:
|
||||
dewies = lbc_to_dewies(lbc)
|
||||
if positive_value and dewies <= 0:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError(f"'{argument}' value must be greater than 0.0")
|
||||
return dewies
|
||||
except ValueError as e:
|
||||
# TODO: use error from lbry.error
|
||||
raise ValueError(f"Invalid value for '{argument}': {e.args[0]}")
|
||||
|
||||
async def resolve(self, accounts, urls, **kwargs):
|
||||
|
||||
@@ -607,6 +607,12 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
|
||||
return self.sout(txs)
|
||||
|
||||
async def blob_clean(self):
|
||||
return await self.out(self.daemon.jsonrpc_blob_clean())
|
||||
|
||||
async def status(self):
|
||||
return await self.out(self.daemon.jsonrpc_status())
|
||||
|
||||
async def resolve(self, uri, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
from typing import List, Type, MutableSequence, MutableMapping, Optional
|
||||
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
|
||||
from lbry.conf import Config, NOT_SET
|
||||
|
||||
from .dewies import dewies_to_lbc
|
||||
@@ -95,7 +95,7 @@ class WalletManager:
|
||||
for wallet in self.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
return wallet
|
||||
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
|
||||
raise WalletNotLoadedError(wallet_id)
|
||||
|
||||
@staticmethod
|
||||
def get_balance(wallet):
|
||||
|
||||
@@ -10,7 +10,7 @@ from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
|
||||
from elasticsearch.helpers import async_streaming_bulk
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
|
||||
from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
@@ -465,6 +465,8 @@ def expand_query(**kwargs):
|
||||
for key, value in kwargs.items():
|
||||
key = key.replace('claim.', '')
|
||||
many = key.endswith('__in') or isinstance(value, list)
|
||||
if many and len(value) > 2048:
|
||||
raise TooManyClaimSearchParametersError(key, 2048)
|
||||
if many:
|
||||
key = key.replace('__in', '')
|
||||
value = list(filter(None, value))
|
||||
|
||||
@@ -22,19 +22,29 @@ async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
|
||||
db.row_factory = namedtuple_factory
|
||||
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
|
||||
for num, claim in enumerate(db.execute(f"""
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
""")):
|
||||
claim = dict(claim._asdict())
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_id'] = None
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
|
||||
@@ -818,10 +818,15 @@ class SQLDB:
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
|
||||
"""
|
||||
for claim in self.execute(query):
|
||||
@@ -830,6 +835,11 @@ class SQLDB:
|
||||
claim['censor_type'] = 0
|
||||
censoring_channel_hash = None
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
for reason_id in id_set:
|
||||
if reason_id in self.blocked_streams:
|
||||
claim['censor_type'] = 2
|
||||
|
||||
@@ -15,14 +15,13 @@ from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from elasticsearch import ConnectionTimeout
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.error import TooManyClaimSearchParametersError
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
@@ -1028,7 +1027,13 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
try:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
except TooManyClaimSearchParametersError as err:
|
||||
await asyncio.sleep(2)
|
||||
self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.",
|
||||
self.peer_address()[0], err.key, err.limit)
|
||||
return RPCError(1, str(err))
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
|
||||
@@ -304,13 +304,15 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc')
|
||||
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc')
|
||||
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd')
|
||||
repost1 = await self.stream_repost(self.get_claim_id(claim1), 'repost1')
|
||||
repost5 = await self.stream_repost(self.get_claim_id(claim5), 'repost5')
|
||||
|
||||
await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
|
||||
await self.assertFindsClaims([claim4, claim3, claim2, claim1], fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
|
||||
await self.assertFindsClaims([repost5, repost1, claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
|
||||
await self.assertFindsClaims([repost1, claim4, claim3, claim2, claim1], fee_currency='lbc')
|
||||
await self.assertFindsClaims([repost1, claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim5], fee_currency='usd')
|
||||
await self.assertFindsClaims([repost5, claim5], fee_currency='usd')
|
||||
|
||||
async def test_search_by_language(self):
|
||||
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
|
||||
@@ -476,7 +478,8 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
image_repost = await self.stream_repost(self.get_claim_id(image), 'image-repost')
|
||||
video_repost = await self.stream_repost(self.get_claim_id(video), 'video-repost')
|
||||
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
@@ -484,25 +487,25 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
await self.assertFindsClaims([video_repost, image_repost], claim_type='repost')
|
||||
await self.assertFindsClaims([collection], claim_type='collection')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
await self.assertFindsClaims([video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
await self.assertFindsClaims([video_repost, video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image_repost, image], stream_types=['image'])
|
||||
await self.assertFindsClaims([video_repost, image_repost, image, video], stream_types=['video', 'image'])
|
||||
|
||||
# media_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
|
||||
await self.assertFindsClaims([video_repost, video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image_repost, image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([video_repost, image_repost, image, video], media_types=['video/mp4', 'image/png'])
|
||||
|
||||
# duration
|
||||
await self.assertFindsClaim(video, duration='>14')
|
||||
await self.assertFindsClaim(video, duration='<16')
|
||||
await self.assertFindsClaim(video, duration=15)
|
||||
await self.assertFindsClaims([video_repost, video], duration='>14')
|
||||
await self.assertFindsClaims([video_repost, video], duration='<16')
|
||||
await self.assertFindsClaims([video_repost, video], duration=15)
|
||||
await self.assertFindsClaims([], duration='>100')
|
||||
await self.assertFindsClaims([], duration='<14')
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.rpc import RPCError
|
||||
from lbry.wallet.server.db.elasticsearch.sync import run as run_sync, make_es_index
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
@@ -191,7 +192,7 @@ class TestHubDiscovery(CommandTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestStressFlush(CommandTestCase):
|
||||
class TestStress(CommandTestCase):
|
||||
async def test_flush_over_66_thousand(self):
|
||||
history = self.conductor.spv_node.server.db.history
|
||||
history.flush_count = 66_000
|
||||
@@ -199,3 +200,9 @@ class TestStressFlush(CommandTestCase):
|
||||
self.assertEqual(history.flush_count, 66_001)
|
||||
await self.generate(1)
|
||||
self.assertEqual(history.flush_count, 66_002)
|
||||
|
||||
async def test_thousands_claim_ids_on_search(self):
|
||||
await self.stream_create()
|
||||
with self.assertRaises(RPCError) as err:
|
||||
await self.claim_search(not_channel_ids=[("%040x" % i) for i in range(8196)])
|
||||
self.assertEqual(err.exception.message, 'not_channel_ids cant be set for more than 2048 items.')
|
||||
@@ -1,10 +1,11 @@
|
||||
from unittest import skipIf
|
||||
import asyncio
|
||||
import os
|
||||
from binascii import hexlify
|
||||
|
||||
from lbry.schema import Claim
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.extras.daemon.components import TorrentSession
|
||||
from lbry.wallet import Transaction
|
||||
|
||||
|
||||
@@ -40,6 +41,7 @@ class FileCommands(CommandTestCase):
|
||||
self.client_session.wait_start = False # fixme: this is super slow on tests
|
||||
return tx, btih
|
||||
|
||||
@skipIf(TorrentSession is None, "libtorrent not installed")
|
||||
async def test_download_torrent(self):
|
||||
tx, btih = await self.initialize_torrent()
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
@@ -509,3 +511,20 @@ class FileCommands(CommandTestCase):
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
|
||||
class DiskSpaceManagement(CommandTestCase):
|
||||
|
||||
async def test_file_management(self):
|
||||
status = await self.status()
|
||||
self.assertIn('disk_space', status)
|
||||
self.assertEqual('0', status['disk_space']['space_used'])
|
||||
self.assertEqual(True, status['disk_space']['running'])
|
||||
await self.stream_create('foo1', '0.01', data=('0' * 3 * 1024 * 1024).encode())
|
||||
await self.stream_create('foo2', '0.01', data=('0' * 2 * 1024 * 1024).encode())
|
||||
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
|
||||
await self.blob_clean()
|
||||
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
|
||||
self.daemon.conf.blob_storage_limit = 3
|
||||
await self.blob_clean()
|
||||
self.assertEqual('3', (await self.status())['disk_space']['space_used'])
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
import unittest
|
||||
import tempfile
|
||||
|
||||
import lbry.wallet
|
||||
from lbry.conf import Config
|
||||
from lbry.blob.disk_space_manager import DiskSpaceManager
|
||||
|
||||
|
||||
class ConfigurationTests(unittest.TestCase):
|
||||
|
||||
def test_space_management(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
os.mkdir(os.path.join(temp_dir, 'blobfiles'))
|
||||
config = Config(
|
||||
blob_storage_limit=5,
|
||||
data_dir=temp_dir,
|
||||
wallet_dir=temp_dir,
|
||||
config=os.path.join(temp_dir, 'settings.yml'),
|
||||
)
|
||||
dsm = DiskSpaceManager(config)
|
||||
self.assertEqual(0, dsm.space_used_mb)
|
||||
for file_no in range(10):
|
||||
with open(os.path.join(config.data_dir, 'blobfiles', f'3mb-{file_no}'), 'w') as blob:
|
||||
blob.write('0' * 1 * 1024 * 1024)
|
||||
self.assertEqual(10, dsm.space_used_mb)
|
||||
self.assertTrue(dsm.clean())
|
||||
self.assertEqual(5, dsm.space_used_mb)
|
||||
self.assertFalse(dsm.clean())
|
||||
|
||||
@@ -15,6 +15,7 @@ class TestComponentManager(AsyncioTestCase):
|
||||
self.default_components_sort = [
|
||||
[
|
||||
components.DatabaseComponent,
|
||||
components.DiskSpaceComponent,
|
||||
components.ExchangeRateManagerComponent,
|
||||
components.TorrentComponent,
|
||||
components.UPnPComponent
|
||||
|
||||
@@ -21,6 +21,27 @@ class TestConfig(BaseConfig):
|
||||
|
||||
class ConfigurationTests(unittest.TestCase):
|
||||
|
||||
@unittest.skipIf('darwin' not in sys.platform, 'skipping mac only test')
|
||||
def test_mac_defaults(self):
|
||||
c = Config()
|
||||
self.assertEqual(c.data_dir, os.path.expanduser("~/Library/Application Support/LBRY"))
|
||||
self.assertEqual(c.wallet_dir, os.path.expanduser('~/.lbryum'))
|
||||
self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads'))
|
||||
self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml'))
|
||||
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
|
||||
self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log'))
|
||||
|
||||
@unittest.skipIf('win32' not in sys.platform, 'skipping windows only test')
|
||||
def test_windows_defaults(self):
|
||||
c = Config()
|
||||
prefix = os.path.join(r"C:\Users", os.getlogin(), r"AppData\Local\lbry")
|
||||
self.assertEqual(c.data_dir, os.path.join(prefix, 'lbrynet'))
|
||||
self.assertEqual(c.wallet_dir, os.path.join(prefix, 'lbryum'))
|
||||
self.assertEqual(c.download_dir, os.path.join(r"C:\Users", os.getlogin(), "Downloads"))
|
||||
self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml'))
|
||||
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
|
||||
self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log'))
|
||||
|
||||
@unittest.skipIf('linux' not in sys.platform, 'skipping linux only test')
|
||||
def test_linux_defaults(self):
|
||||
c = Config()
|
||||
|
||||
Reference in new issue
Block a user