Compare commits

...
36 Commits
Author SHA1 Message Date
Lex Berezhny 7582c221d1 v0.102.0
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
2021-08-17 14:16:17 -04:00
Lex Berezhny c109895848 Merge pull request #3399 from lbryio/better-error-logging
Less verbose error logs, only log tracebacks for errors not defined in `lbry.error`
2021-08-17 14:14:27 -04:00
Jack Robison eccedada40 add TODOs for errors raised that aren't defined in lbry.error 2021-08-17 12:31:03 -04:00
Jack Robison 25d54accf8 return api errors from wallet_add and wallet_create 2021-08-17 12:30:17 -04:00
Jack Robison d07685f0e9 only log tracebacks for api errors not defined in lbry.error 2021-08-17 11:30:58 -04:00
Jack Robison 2445c00c7e raise WalletNotLoadedError in get_wallet_or_error instead of ValueError 2021-08-17 11:30:58 -04:00
Lex Berezhny 4c1d3ef514 Merge pull request #3398 from lbryio/clean_blobs_after_delay
clean blobs after waiting interval instead of immediately on startup
2021-08-17 10:44:34 -04:00
Lex Berezhny 4614c7d4c2 clean blobs after waiting interval instead of immediately on startup 2021-08-17 09:52:44 -04:00
Lex Berezhny bbf1ef0dc3 Merge pull request #3378 from lbryio/disk_management
ability to limit disk spaced used for blobs via `blob_storage_limit` setting (oldest blobs are deleted when disk space limit is reached)
2021-08-16 17:41:05 -04:00
Lex Berezhny 3433c9e708 return number of files deleted 2021-08-16 17:03:40 -04:00
Lex Berezhny 2cd5d75a2e return true/false if clean was performed 2021-08-16 17:02:13 -04:00
Lex Berezhny 2535b8adef fix disk space unit test 2021-08-16 14:54:17 -04:00
Lex Berezhny 4edab7bb7f fix sorting by DirEntry error 2021-08-16 14:41:16 -04:00
Lex Berezhny fd8658e317 test component unit test 2021-08-16 14:35:32 -04:00
Lex Berezhny 51d21d8c86 working disk cleanup 2021-08-16 14:15:12 -04:00
Lex Berezhny b4c3307cdf fixed tests 2021-08-13 10:32:46 -04:00
Lex Berezhny 4e8d10cb44 disk space manager and status API 2021-08-13 10:32:46 -04:00
Lex Berezhny e96875a425 workflow syntax 2021-08-13 10:32:46 -04:00
Lex Berezhny 5ab0035348 run tests on windows and mac 2021-08-13 10:32:46 -04:00
Lex Berezhny 4ddff96b1e Merge pull request #3395 from lbryio/libtorrent_optional
make libtorrent optional and skip test which depends on it
2021-08-13 10:31:06 -04:00
Lex Berezhny a08d84c1df make libtorrent optional and skip test which depends on it 2021-08-13 10:07:06 -04:00
Victor Shyba 21c71bfac1 update sync utility 2021-08-09 18:33:47 -03:00
Victor Shyba 6baaed3581 refactor query with new fields 2021-08-09 18:33:47 -03:00
Victor Shyba 152dbfd5d1 reflect fee_currency, fee_amount and duration on repost searches 2021-08-09 18:33:47 -03:00
Victor Shyba a56d14086b reflect media_type on repost searches 2021-08-09 18:33:47 -03:00
Victor Shyba aee87693f8 reflect stream_type on repost searches 2021-08-09 18:33:47 -03:00
Alex Grin 976b4affd9 Merge pull request #3383 from lbryio/dht_log 2021-08-09 17:10:01 -04:00
Victor Shyba e222b6ad9c log that a invalid query happened 2021-08-09 15:07:44 -03:00
Victor Shyba 19b17374e8 throttle instead of disconnecting 2021-08-09 15:07:44 -03:00
Victor Shyba 43989122bb add error type and message to error readme and update code 2021-08-09 15:07:44 -03:00
Victor Shyba 72712d6047 raise and disconnect if too many parameters are used on search 2021-08-09 15:07:44 -03:00
Victor Shyba 0b52d2cc15 log invalid port as a warning instead of an exception 2021-08-03 15:29:52 -03:00
Lex Berezhny 8304102136 move <!channel> out of markdown converter 2021-07-27 11:54:08 -04:00
Lex Berezhny 3381aefcfa notify channel of slack message 2021-07-27 11:39:35 -04:00
Lex Berezhny 279a365cb1 v0.101.1
ci / lint (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
2021-07-27 11:12:06 -04:00
Lex Berezhny 2c9e00da56 revert version 2021-07-27 11:10:16 -04:00
23 changed files with 398 additions and 56 deletions

No files matched your search

+12 -4
View File
@@ -31,8 +31,8 @@ jobs:
matrix:
os:
- ubuntu-latest
#- macos-latest
#- windows-latest
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
@@ -49,10 +49,18 @@ jobs:
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: pip install --user --upgrade pip wheel
- run: pip install -e .[torrent,test]
- env:
- if: startsWith(runner.os, 'linux')
run: pip install -e .[torrent,test]
- if: startsWith(runner.os, 'linux')
env:
HOME: /tmp
run: make test-unit-coverage
- if: startsWith(runner.os, 'linux') != true
run: pip install -e .[test]
- if: startsWith(runner.os, 'linux') != true
env:
HOME: /tmp
run: python -m unittest tests/unit/test_conf.py
tests-integration:
name: "tests / integration"
+2 -2
View File
@@ -15,8 +15,8 @@ jobs:
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
- uses: slackapi/slack-github-action@v1.14.0
env:
CHANGELOG: ${{ toJSON(steps.markdown.outputs.text) }}
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
with:
payload: '{"type": "mrkdwn", "text": ${{ env.CHANGELOG }}}'
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }}'
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.101.1"
__version__ = "0.102.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+64
View File
@@ -0,0 +1,64 @@
import os
import asyncio
import logging
log = logging.getLogger(__name__)
class DiskSpaceManager:
def __init__(self, config, cleaning_interval=30 * 60):
self.config = config
self.cleaning_interval = cleaning_interval
self.running = False
self.task = None
@property
def space_used_bytes(self):
used = 0
data_dir = os.path.join(self.config.data_dir, 'blobfiles')
for item in os.scandir(data_dir):
if item.is_file:
used += item.stat().st_size
return used
@property
def space_used_mb(self):
return int(self.space_used_bytes/1024.0/1024.0)
def clean(self):
if not self.config.blob_storage_limit:
return 0
used = 0
files = []
data_dir = os.path.join(self.config.data_dir, 'blobfiles')
for file in os.scandir(data_dir):
if file.is_file:
file_stats = file.stat()
used += file_stats.st_size
files.append((file_stats.st_mtime, file_stats.st_size, file.path))
files.sort()
available = (self.config.blob_storage_limit*1024*1024) - used
cleaned = 0
for _, file_size, file in files:
available += file_size
if available > 0:
break
os.remove(file)
cleaned += 1
return cleaned
async def cleaning_loop(self):
while self.running:
await asyncio.sleep(self.cleaning_interval)
await asyncio.get_event_loop().run_in_executor(None, self.clean)
async def start(self):
self.running = True
self.task = asyncio.create_task(self.cleaning_loop())
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
async def stop(self):
if self.running:
self.running = False
self.task.cancel()
+1
View File
@@ -634,6 +634,7 @@ class Config(CLIConfig):
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32
+2 -2
View File
@@ -156,9 +156,9 @@ class KademliaPeer:
if not len(self._node_id) == constants.HASH_LENGTH:
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
raise ValueError("invalid udp port")
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
raise ValueError("invalid tcp port")
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
if not is_valid_public_ipv4(self.address, self.allow_localhost):
raise ValueError(f"invalid ip address: '{self.address}'")
+5 -1
View File
@@ -452,7 +452,11 @@ class KademliaProtocol(DatagramProtocol):
try:
peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError:
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
except ValueError as err:
log.warning("error replying to %s: %s", address[0], str(err))
return
try:
self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
+6 -1
View File
@@ -53,13 +53,18 @@ Code | Name | Message
407 | DataDownload | Failed to download blob. *generic*
410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
420 | KeyFeeAboveMaxAllowed | {message}
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
423 | TooManyClaimSearchParameters | {key} cant be set for more than {limit} items.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
436 | WalletNotFound | Wallet not found at {wallet_path}.
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
**5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob.
+36
View File
@@ -242,6 +242,14 @@ class IncompatibleWalletServerError(WalletError):
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class TooManyClaimSearchParametersError(WalletError):
def __init__(self, key, limit):
self.key = key
self.limit = limit
super().__init__(f"{key} cant be set for more than {limit} items.")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
@@ -263,6 +271,34 @@ class ServerPaymentFeeAboveMaxAllowedError(WalletError):
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
class WalletNotLoadedError(WalletError):
def __init__(self, wallet_id):
self.wallet_id = wallet_id
super().__init__(f"Wallet {wallet_id} is not loaded.")
class WalletAlreadyLoadedError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet {wallet_path} is already loaded.")
class WalletNotFoundError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet not found at {wallet_path}.")
class WalletAlreadyExistsError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
class BlobError(BaseError):
"""
**Blobs**
+26
View File
@@ -15,6 +15,7 @@ from lbry.dht.node import Node
from lbry.dht.peer import is_valid_public_ipv4
from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.blob.blob_manager import BlobManager
from lbry.blob.disk_space_manager import DiskSpaceManager
from lbry.blob_exchange.server import BlobServer
from lbry.stream.stream_manager import StreamManager
from lbry.file.file_manager import FileManager
@@ -40,6 +41,7 @@ WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
FILE_MANAGER_COMPONENT = "file_manager"
DISK_SPACE_COMPONENT = "disk_space"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
@@ -375,6 +377,30 @@ class FileManagerComponent(Component):
self.file_manager.stop()
class DiskSpaceComponent(Component):
component_name = DISK_SPACE_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.disk_space_manager = DiskSpaceManager(self.conf)
@property
def component(self) -> typing.Optional[DiskSpaceManager]:
return self.disk_space_manager
async def get_status(self):
return {
'space_used': str(self.disk_space_manager.space_used_mb),
'running': self.disk_space_manager.running,
}
async def start(self):
await self.disk_space_manager.start()
async def stop(self):
await self.disk_space_manager.stop()
class TorrentComponent(Component):
component_name = LIBTORRENT_COMPONENT
+93 -9
View File
@@ -36,12 +36,12 @@ from lbry.blob_exchange.downloader import download_blob
from lbry.dht.peer import make_kademlia_peer
from lbry.error import (
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
CommandDoesNotExistError
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError
)
from lbry.extras import system_info
from lbry.extras.daemon import analytics
from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT
from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT
from lbry.extras.daemon.componentmanager import RequiredCondition
from lbry.extras.daemon.componentmanager import ComponentManager
@@ -56,7 +56,7 @@ from lbry.schema.url import URL
if typing.TYPE_CHECKING:
from lbry.blob.blob_manager import BlobManager
from lbry.dht.node import Node
from lbry.extras.daemon.components import UPnPComponent
from lbry.extras.daemon.components import UPnPComponent, DiskSpaceManager
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager, Ledger
@@ -375,6 +375,10 @@ class Daemon(metaclass=JSONRPCServerType):
def blob_manager(self) -> typing.Optional['BlobManager']:
return self.component_manager.get_component(BLOB_COMPONENT)
@property
def disk_space_manager(self) -> typing.Optional['DiskSpaceManager']:
return self.component_manager.get_component(DISK_SPACE_COMPONENT)
@property
def upnp(self) -> typing.Optional['UPnPComponent']:
return self.component_manager.get_component(UPNP_COMPONENT)
@@ -698,7 +702,10 @@ class Daemon(metaclass=JSONRPCServerType):
raise
except Exception as e: # pylint: disable=broad-except
self.failed_request_metric.labels(method=function_name).inc()
log.exception("error handling api request")
if not isinstance(e, BaseError):
log.exception("error handling api request")
else:
log.error("error handling api request: %s", e)
return JSONRPCError.create_command_exception(
command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc()
)
@@ -1103,6 +1110,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not stream:
raise DownloadSDTimeoutError(uri)
except Exception as e:
# TODO: use error from lbry.error
log.warning("Error downloading %s: %s", uri, str(e))
return {"error": str(e)}
return stream
@@ -1269,9 +1277,9 @@ class Daemon(metaclass=JSONRPCServerType):
wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id)
for wallet in self.wallet_manager.wallets:
if wallet.id == wallet_id:
raise Exception(f"Wallet at path '{wallet_path}' already exists and is loaded.")
raise WalletAlreadyLoadedError(wallet_path)
if os.path.exists(wallet_path):
raise Exception(f"Wallet at path '{wallet_path}' already exists, use 'wallet_add' to load wallet.")
raise WalletAlreadyExistsError(wallet_path)
wallet = self.wallet_manager.import_wallet(wallet_path)
if not wallet.accounts and create_account:
@@ -1304,9 +1312,9 @@ class Daemon(metaclass=JSONRPCServerType):
wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id)
for wallet in self.wallet_manager.wallets:
if wallet.id == wallet_id:
raise Exception(f"Wallet at path '{wallet_path}' is already loaded.")
raise WalletAlreadyLoadedError(wallet_path)
if not os.path.exists(wallet_path):
raise Exception(f"Wallet at path '{wallet_path}' was not found.")
raise WalletNotFoundError(wallet_path)
wallet = self.wallet_manager.import_wallet(wallet_path)
if self.ledger.network.is_connected:
for account in wallet.accounts:
@@ -1491,7 +1499,7 @@ class Daemon(metaclass=JSONRPCServerType):
)
)
else:
raise ValueError(f"Unsupported address: '{address}'")
raise ValueError(f"Unsupported address: '{address}'") # TODO: use error from lbry.error
tx = await Transaction.create(
[], outputs, accounts, account
@@ -1769,8 +1777,10 @@ class Daemon(metaclass=JSONRPCServerType):
from_account = wallet.get_account_or_default(from_account)
amount = self.get_dewies_or_error('amount', amount) if amount else None
if not isinstance(outputs, int):
# TODO: use error from lbry.error
raise ValueError("--outputs must be an integer.")
if everything and outputs > 1:
# TODO: use error from lbry.error
raise ValueError("Using --everything along with --outputs is not supported.")
return from_account.fund(
to_account=to_account, amount=amount, everything=everything,
@@ -2041,10 +2051,12 @@ class Daemon(metaclass=JSONRPCServerType):
"""
if status not in ['start', 'stop']:
# TODO: use error from lbry.error
raise Exception('Status must be "start" or "stop".')
streams = self.file_manager.get_filtered(**kwargs)
if not streams:
# TODO: use error from lbry.error
raise Exception(f'Unable to find a file for {kwargs}')
stream = streams[0]
if status == 'start' and not stream.running:
@@ -2227,20 +2239,25 @@ class Daemon(metaclass=JSONRPCServerType):
if claim_id:
txo = await self.ledger.get_claim_by_claim_id(accounts, claim_id, include_purchase_receipt=True)
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find claim with claim_id '{claim_id}'. ")
elif url:
txo = (await self.ledger.resolve(accounts, [url], include_purchase_receipt=True))[url]
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find claim with url '{url}'. ")
else:
# TODO: use error from lbry.error
raise Exception(f"Missing argument claim_id or url. ")
if not allow_duplicate_purchase and txo.purchase_receipt:
# TODO: use error from lbry.error
raise Exception(
f"You already have a purchase for claim_id '{claim_id}'. "
f"Use --allow-duplicate-purchase flag to override."
)
claim = txo.claim
if not claim.is_stream or not claim.stream.has_fee:
# TODO: use error from lbry.error
raise Exception(f"Claim '{claim_id}' does not have a purchase price.")
tx = await self.wallet_manager.create_purchase_transaction(
accounts, txo, self.exchange_rate_manager, override_max_key_fee
@@ -2482,6 +2499,7 @@ class Daemon(metaclass=JSONRPCServerType):
if "claim_ids" in kwargs and not kwargs["claim_ids"]:
kwargs.pop("claim_ids")
if {'claim_id', 'claim_ids'}.issubset(kwargs):
# TODO: use error from lbry.error
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
if kwargs.pop('valid_channel_signature', False):
kwargs['signature_valid'] = 1
@@ -2601,6 +2619,7 @@ class Daemon(metaclass=JSONRPCServerType):
existing_channels = await self.ledger.get_channels(accounts=wallet.accounts, claim_name=name)
if len(existing_channels) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a channel under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
@@ -2733,11 +2752,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(existing_channels) != 1:
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
# TODO: use error from lbry.error
raise Exception(
f"Can't find the channel '{claim_id}' in account(s) {account_ids}."
)
old_txo = existing_channels[0]
if not old_txo.claim.is_channel:
# TODO: use error from lbry.error
raise Exception(
f"A claim with id '{claim_id}' was found but it is not a channel."
)
@@ -2860,9 +2881,11 @@ class Daemon(metaclass=JSONRPCServerType):
wallet=wallet, accounts=accounts, claim_id=claim_id
)
else:
# TODO: use error from lbry.error
raise Exception('Must specify claim_id, or txid and nout')
if not claims:
# TODO: use error from lbry.error
raise Exception('No claim found for the specified claim_id or txid:nout')
tx = await Transaction.create(
@@ -2931,6 +2954,7 @@ class Daemon(metaclass=JSONRPCServerType):
address = channel.get_address(self.ledger)
public_key = await self.ledger.get_public_key_for_address(wallet, address)
if not public_key:
# TODO: use error from lbry.error
raise Exception("Can't find public key for address holding the channel.")
export = {
'name': channel.claim_name,
@@ -2992,6 +3016,7 @@ class Daemon(metaclass=JSONRPCServerType):
await self.ledger._update_tasks.done.wait()
# Case 3: the holding address has changed and we can't create or find an account for it
else:
# TODO: use error from lbry.error
raise Exception(
"Channel owning account has changed since the channel was exported and "
"it is not an account to which you have access."
@@ -3117,11 +3142,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(claims) == 0:
if 'bid' not in kwargs:
# TODO: use error from lbry.error
raise Exception("'bid' is a required argument for new publishes.")
return await self.jsonrpc_stream_create(name, **kwargs)
elif len(claims) == 1:
assert claims[0].claim.is_stream, f"Claim at name '{name}' is not a stream claim."
return await self.jsonrpc_stream_update(claims[0].claim_id, replace=True, **kwargs)
# TODO: use error from lbry.error
raise Exception(
f"There are {len(claims)} claims for '{name}', please use 'stream update' command "
f"to update a specific stream claim."
@@ -3173,11 +3200,13 @@ class Daemon(metaclass=JSONRPCServerType):
claims = await account.get_claims(claim_name=name)
if len(claims) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a stream claim published under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
)
if not VALID_FULL_CLAIM_ID.fullmatch(claim_id):
# TODO: use error from lbry.error
raise Exception('Invalid claim id. It is expected to be a 40 characters long hexadecimal string.')
claim = Claim()
@@ -3321,6 +3350,7 @@ class Daemon(metaclass=JSONRPCServerType):
claims = await account.get_claims(claim_name=name)
if len(claims) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a stream claim published under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
@@ -3504,11 +3534,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(existing_claims) != 1:
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
# TODO: use error from lbry.error
raise Exception(
f"Can't find the stream '{claim_id}' in account(s) {account_ids}."
)
old_txo = existing_claims[0]
if not old_txo.claim.is_stream:
# TODO: use error from lbry.error
raise Exception(
f"A claim with id '{claim_id}' was found but it is not a stream claim."
)
@@ -3636,9 +3668,11 @@ class Daemon(metaclass=JSONRPCServerType):
wallet=wallet, accounts=accounts, claim_id=claim_id
)
else:
# TODO: use error from lbry.error
raise Exception('Must specify claim_id, or txid and nout')
if not claims:
# TODO: use error from lbry.error
raise Exception('No claim found for the specified claim_id or txid:nout')
tx = await Transaction.create(
@@ -3800,6 +3834,7 @@ class Daemon(metaclass=JSONRPCServerType):
existing_collections = await self.ledger.get_collections(accounts=wallet.accounts, claim_name=name)
if len(existing_collections) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a collection under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
@@ -3923,11 +3958,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(existing_collections) != 1:
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
# TODO: use error from lbry.error
raise Exception(
f"Can't find the collection '{claim_id}' in account(s) {account_ids}."
)
old_txo = existing_collections[0]
if not old_txo.claim.is_collection:
# TODO: use error from lbry.error
raise Exception(
f"A claim with id '{claim_id}' was found but it is not a collection."
)
@@ -4053,12 +4090,15 @@ class Daemon(metaclass=JSONRPCServerType):
if claim_id:
txo = await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id)
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find collection with claim_id '{claim_id}'. ")
elif url:
txo = (await self.ledger.resolve(wallet.accounts, [url]))[url]
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find collection with url '{url}'. ")
else:
# TODO: use error from lbry.error
raise Exception(f"Missing argument claim_id or url. ")
page_num, page_size = abs(page), min(abs(page_size), 50)
@@ -4233,9 +4273,11 @@ class Daemon(metaclass=JSONRPCServerType):
wallet=wallet, accounts=accounts, claim_id=claim_id
)
else:
# TODO: use error from lbry.error
raise Exception('Must specify claim_id, or txid and nout')
if not supports:
# TODO: use error from lbry.error
raise Exception('No supports found for the specified claim_id or txid:nout')
if keep is not None:
@@ -4470,6 +4512,7 @@ class Daemon(metaclass=JSONRPCServerType):
elif order_by in ('height', 'amount', 'none'):
constraints['order_by'] = order_by
else:
# TODO: use error from lbry.error
raise ValueError(f"'{order_by}' is not a valid --order_by value.")
self._constrain_txo_from_kwargs(constraints, **kwargs)
return paginate_rows(claims, None if no_totals else claim_count, page, page_size, **constraints)
@@ -4776,10 +4819,12 @@ class Daemon(metaclass=JSONRPCServerType):
"""
if not is_valid_blobhash(blob_hash):
# TODO: use error from lbry.error
raise Exception("invalid blob hash")
if search_bottom_out_limit is not None:
search_bottom_out_limit = int(search_bottom_out_limit)
if search_bottom_out_limit <= 0:
# TODO: use error from lbry.error
raise Exception("invalid bottom out limit")
else:
search_bottom_out_limit = 4
@@ -4823,12 +4868,14 @@ class Daemon(metaclass=JSONRPCServerType):
blob_hashes.append(blob_hash)
elif stream_hash or sd_hash:
if sd_hash and stream_hash:
# TODO: use error from lbry.error
raise Exception("either the sd hash or the stream hash should be provided, not both")
if sd_hash:
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
blobs = await self.storage.get_blobs_for_stream(stream_hash, only_completed=True)
blob_hashes.extend(blob.blob_hash for blob in blobs if blob.blob_hash is not None)
else:
# TODO: use error from lbry.error
raise Exception('single argument must be specified')
await self.storage.should_single_announce_blobs(blob_hashes, immediate=True)
return True
@@ -4916,6 +4963,22 @@ class Daemon(metaclass=JSONRPCServerType):
raise NotImplementedError()
@requires(DISK_SPACE_COMPONENT)
async def jsonrpc_blob_clean(self):
"""
Deletes blobs to cleanup disk space
Usage:
blob_clean
Options:
None
Returns:
(bool) true if successful
"""
return self.disk_space_manager.clean()
@requires(FILE_MANAGER_COMPONENT)
async def jsonrpc_file_reflect(self, **kwargs):
"""
@@ -5079,6 +5142,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
if not tracemalloc.is_tracing():
# TODO: use error from lbry.error
raise Exception("Enable tracemalloc first! See 'tracemalloc set' command.")
stats = tracemalloc.take_snapshot().filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
@@ -5288,6 +5352,7 @@ class Daemon(metaclass=JSONRPCServerType):
comment_id=comment_id
)
if 'error' in channel:
# TODO: use error from lbry.error
raise ValueError(channel['error'])
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
@@ -5544,51 +5609,64 @@ class Daemon(metaclass=JSONRPCServerType):
allow_script_address and self.ledger.is_script_address(address)
)
except:
# TODO: use error from lbry.error
raise Exception(f"'{address}' is not a valid address")
@staticmethod
def valid_stream_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception('Stream name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
# TODO: use error from lbry.error
raise Exception(
"Stream names cannot start with '@' symbol. This is reserved for channels claims."
)
if not parsed.has_stream or parsed.stream.name != name:
# TODO: use error from lbry.error
raise Exception('Stream name has invalid characters.')
except (TypeError, ValueError):
# TODO: use error from lbry.error
raise Exception("Invalid stream name.")
@staticmethod
def valid_collection_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception('Collection name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
# TODO: use error from lbry.error
raise Exception(
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
)
if not parsed.has_stream or parsed.stream.name != name:
# TODO: use error from lbry.error
raise Exception('Collection name has invalid characters.')
except (TypeError, ValueError):
# TODO: use error from lbry.error
raise Exception("Invalid collection name.")
@staticmethod
def valid_channel_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception(
"Channel name cannot be blank."
)
parsed = URL.parse(name)
if not parsed.has_channel:
# TODO: use error from lbry.error
raise Exception("Channel names must start with '@' symbol.")
if parsed.channel.name != name:
# TODO: use error from lbry.error
raise Exception("Channel name has invalid character")
except (TypeError, ValueError):
# TODO: use error from lbry.error
raise Exception("Invalid channel name.")
def get_fee_address(self, kwargs: dict, claim_address: str) -> str:
@@ -5620,6 +5698,7 @@ class Daemon(metaclass=JSONRPCServerType):
elif channel_name:
key, value = 'name', channel_name
else:
# TODO: use error from lbry.error
raise ValueError("Couldn't find channel because a channel_id or channel_name was not provided.")
channels = await self.ledger.get_channels(
wallet=wallet, accounts=wallet.get_accounts_or_all(account_ids),
@@ -5627,13 +5706,16 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(channels) == 1:
if for_signing and not channels[0].has_private_key:
# TODO: use error from lbry.error
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
return channels[0]
elif len(channels) > 1:
# TODO: use error from lbry.error
raise ValueError(
f"Multiple channels found with channel_{key} '{value}', "
f"pass a channel_id to narrow it down."
)
# TODO: use error from lbry.error
raise ValueError(f"Couldn't find channel with channel_{key} '{value}'.")
@staticmethod
@@ -5641,9 +5723,11 @@ class Daemon(metaclass=JSONRPCServerType):
try:
dewies = lbc_to_dewies(lbc)
if positive_value and dewies <= 0:
# TODO: use error from lbry.error
raise ValueError(f"'{argument}' value must be greater than 0.0")
return dewies
except ValueError as e:
# TODO: use error from lbry.error
raise ValueError(f"Invalid value for '{argument}': {e.args[0]}")
async def resolve(self, accounts, urls, **kwargs):
+6
View File
@@ -607,6 +607,12 @@ class CommandTestCase(IntegrationTestCase):
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
return self.sout(txs)
async def blob_clean(self):
return await self.out(self.daemon.jsonrpc_blob_clean())
async def status(self):
return await self.out(self.daemon.jsonrpc_status())
async def resolve(self, uri, **kwargs):
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
+2 -2
View File
@@ -7,7 +7,7 @@ from binascii import unhexlify
from decimal import Decimal
from typing import List, Type, MutableSequence, MutableMapping, Optional
from lbry.error import KeyFeeAboveMaxAllowedError
from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
from lbry.conf import Config, NOT_SET
from .dewies import dewies_to_lbc
@@ -95,7 +95,7 @@ class WalletManager:
for wallet in self.wallets:
if wallet.id == wallet_id:
return wallet
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
raise WalletNotLoadedError(wallet_id)
@staticmethod
def get_balance(wallet):
@@ -10,7 +10,7 @@ from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
from elasticsearch.helpers import async_streaming_bulk
from lbry.crypto.base58 import Base58
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError
from lbry.schema.result import Outputs, Censor
from lbry.schema.tags import clean_tags
from lbry.schema.url import URL, normalize_name
@@ -465,6 +465,8 @@ def expand_query(**kwargs):
for key, value in kwargs.items():
key = key.replace('claim.', '')
many = key.endswith('__in') or isinstance(value, list)
if many and len(value) > 2048:
raise TooManyClaimSearchParametersError(key, 2048)
if many:
key = key.replace('__in', '')
value = list(filter(None, value))
+20 -10
View File
@@ -22,19 +22,29 @@ async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
db.row_factory = namedtuple_factory
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
for num, claim in enumerate(db.execute(f"""
SELECT claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash)
WHERE claim.height % {shards_total} = {shard_num}
ORDER BY claim.height desc
SELECT claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
cr.has_source as reposted_has_source,
cr.claim_type as reposted_claim_type,
cr.stream_type as reposted_stream_type,
cr.media_type as reposted_media_type,
cr.duration as reposted_duration,
cr.fee_amount as reposted_fee_amount,
cr.fee_currency as reposted_fee_currency,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
WHERE claim.height % {shards_total} = {shard_num}
ORDER BY claim.height desc
""")):
claim = dict(claim._asdict())
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
claim['censor_type'] = 0
claim['censoring_channel_id'] = None
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
+13 -3
View File
@@ -818,10 +818,15 @@ class SQLDB:
claimtrie.last_take_over_height,
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
cr.has_source as reposted_has_source,
cr.claim_type as reposted_claim_type,
cr.stream_type as reposted_stream_type,
cr.media_type as reposted_media_type,
cr.duration as reposted_duration,
cr.fee_amount as reposted_fee_amount,
cr.fee_currency as reposted_fee_currency,
claim.*
FROM claim LEFT JOIN claimtrie USING (claim_hash)
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
"""
for claim in self.execute(query):
@@ -830,6 +835,11 @@ class SQLDB:
claim['censor_type'] = 0
censoring_channel_hash = None
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
for reason_id in id_set:
if reason_id in self.blocked_streams:
claim['censor_type'] = 2
+8 -3
View File
@@ -15,14 +15,13 @@ from asyncio import Event, sleep
from collections import defaultdict
from functools import partial
from binascii import hexlify
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from elasticsearch import ConnectionTimeout
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.utils import LRUCacheWithMetrics
from lbry.error import TooManyClaimSearchParametersError
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
@@ -1028,7 +1027,13 @@ class LBRYElectrumX(SessionBase):
async def claimtrie_search(self, **kwargs):
if kwargs:
return await self.run_and_cache_query('search', kwargs)
try:
return await self.run_and_cache_query('search', kwargs)
except TooManyClaimSearchParametersError as err:
await asyncio.sleep(2)
self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.",
self.peer_address()[0], err.key, err.limit)
return RPCError(1, str(err))
async def claimtrie_resolve(self, *urls):
if urls:
@@ -304,13 +304,15 @@ class ClaimSearchCommand(ClaimTestCase):
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc')
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc')
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd')
repost1 = await self.stream_repost(self.get_claim_id(claim1), 'repost1')
repost5 = await self.stream_repost(self.get_claim_id(claim5), 'repost5')
await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
await self.assertFindsClaims([claim4, claim3, claim2, claim1], fee_currency='lbc')
await self.assertFindsClaims([claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
await self.assertFindsClaims([repost5, repost1, claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
await self.assertFindsClaims([repost1, claim4, claim3, claim2, claim1], fee_currency='lbc')
await self.assertFindsClaims([repost1, claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
await self.assertFindsClaims([claim5], fee_currency='usd')
await self.assertFindsClaims([repost5, claim5], fee_currency='usd')
async def test_search_by_language(self):
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
@@ -476,7 +478,8 @@ class ClaimSearchCommand(ClaimTestCase):
octet = await self.stream_create()
video = await self.stream_create('chrome', file_path=self.video_file_name)
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
repost = await self.stream_repost(self.get_claim_id(image))
image_repost = await self.stream_repost(self.get_claim_id(image), 'image-repost')
video_repost = await self.stream_repost(self.get_claim_id(video), 'video-repost')
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
channel = await self.channel_create()
unknown = self.sout(tx)
@@ -484,25 +487,25 @@ class ClaimSearchCommand(ClaimTestCase):
# claim_type
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
await self.assertFindsClaims([repost], claim_type='repost')
await self.assertFindsClaims([video_repost, image_repost], claim_type='repost')
await self.assertFindsClaims([collection], claim_type='collection')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
await self.assertFindsClaims([video], stream_types=['video'])
await self.assertFindsClaims([image], stream_types=['image'])
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
await self.assertFindsClaims([video_repost, video], stream_types=['video'])
await self.assertFindsClaims([image_repost, image], stream_types=['image'])
await self.assertFindsClaims([video_repost, image_repost, image, video], stream_types=['video', 'image'])
# media_type
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
await self.assertFindsClaims([video], media_types=['video/mp4'])
await self.assertFindsClaims([image], media_types=['image/png'])
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
await self.assertFindsClaims([video_repost, video], media_types=['video/mp4'])
await self.assertFindsClaims([image_repost, image], media_types=['image/png'])
await self.assertFindsClaims([video_repost, image_repost, image, video], media_types=['video/mp4', 'image/png'])
# duration
await self.assertFindsClaim(video, duration='>14')
await self.assertFindsClaim(video, duration='<16')
await self.assertFindsClaim(video, duration=15)
await self.assertFindsClaims([video_repost, video], duration='>14')
await self.assertFindsClaims([video_repost, video], duration='<16')
await self.assertFindsClaims([video_repost, video], duration=15)
await self.assertFindsClaims([], duration='>100')
await self.assertFindsClaims([], duration='<14')
@@ -4,6 +4,7 @@ import lbry
import lbry.wallet
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.wallet.rpc import RPCError
from lbry.wallet.server.db.elasticsearch.sync import run as run_sync, make_es_index
from lbry.wallet.server.session import LBRYElectrumX
from lbry.testcase import IntegrationTestCase, CommandTestCase
@@ -191,7 +192,7 @@ class TestHubDiscovery(CommandTestCase):
)
class TestStressFlush(CommandTestCase):
class TestStress(CommandTestCase):
async def test_flush_over_66_thousand(self):
history = self.conductor.spv_node.server.db.history
history.flush_count = 66_000
@@ -199,3 +200,9 @@ class TestStressFlush(CommandTestCase):
self.assertEqual(history.flush_count, 66_001)
await self.generate(1)
self.assertEqual(history.flush_count, 66_002)
async def test_thousands_claim_ids_on_search(self):
await self.stream_create()
with self.assertRaises(RPCError) as err:
await self.claim_search(not_channel_ids=[("%040x" % i) for i in range(8196)])
self.assertEqual(err.exception.message, 'not_channel_ids cant be set for more than 2048 items.')
@@ -1,10 +1,11 @@
from unittest import skipIf
import asyncio
import os
from binascii import hexlify
from lbry.schema import Claim
from lbry.testcase import CommandTestCase
from lbry.torrent.session import TorrentSession
from lbry.extras.daemon.components import TorrentSession
from lbry.wallet import Transaction
@@ -40,6 +41,7 @@ class FileCommands(CommandTestCase):
self.client_session.wait_start = False # fixme: this is super slow on tests
return tx, btih
@skipIf(TorrentSession is None, "libtorrent not installed")
async def test_download_torrent(self):
tx, btih = await self.initialize_torrent()
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
@@ -509,3 +511,20 @@ class FileCommands(CommandTestCase):
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
class DiskSpaceManagement(CommandTestCase):
async def test_file_management(self):
status = await self.status()
self.assertIn('disk_space', status)
self.assertEqual('0', status['disk_space']['space_used'])
self.assertEqual(True, status['disk_space']['running'])
await self.stream_create('foo1', '0.01', data=('0' * 3 * 1024 * 1024).encode())
await self.stream_create('foo2', '0.01', data=('0' * 2 * 1024 * 1024).encode())
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
await self.blob_clean()
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
self.daemon.conf.blob_storage_limit = 3
await self.blob_clean()
self.assertEqual('3', (await self.status())['disk_space']['space_used'])
@@ -0,0 +1,30 @@
import os
import unittest
import tempfile
import lbry.wallet
from lbry.conf import Config
from lbry.blob.disk_space_manager import DiskSpaceManager
class ConfigurationTests(unittest.TestCase):
def test_space_management(self):
with tempfile.TemporaryDirectory() as temp_dir:
os.mkdir(os.path.join(temp_dir, 'blobfiles'))
config = Config(
blob_storage_limit=5,
data_dir=temp_dir,
wallet_dir=temp_dir,
config=os.path.join(temp_dir, 'settings.yml'),
)
dsm = DiskSpaceManager(config)
self.assertEqual(0, dsm.space_used_mb)
for file_no in range(10):
with open(os.path.join(config.data_dir, 'blobfiles', f'3mb-{file_no}'), 'w') as blob:
blob.write('0' * 1 * 1024 * 1024)
self.assertEqual(10, dsm.space_used_mb)
self.assertTrue(dsm.clean())
self.assertEqual(5, dsm.space_used_mb)
self.assertFalse(dsm.clean())
@@ -15,6 +15,7 @@ class TestComponentManager(AsyncioTestCase):
self.default_components_sort = [
[
components.DatabaseComponent,
components.DiskSpaceComponent,
components.ExchangeRateManagerComponent,
components.TorrentComponent,
components.UPnPComponent
+21
View File
@@ -21,6 +21,27 @@ class TestConfig(BaseConfig):
class ConfigurationTests(unittest.TestCase):
@unittest.skipIf('darwin' not in sys.platform, 'skipping mac only test')
def test_mac_defaults(self):
c = Config()
self.assertEqual(c.data_dir, os.path.expanduser("~/Library/Application Support/LBRY"))
self.assertEqual(c.wallet_dir, os.path.expanduser('~/.lbryum'))
self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads'))
self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml'))
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log'))
@unittest.skipIf('win32' not in sys.platform, 'skipping windows only test')
def test_windows_defaults(self):
c = Config()
prefix = os.path.join(r"C:\Users", os.getlogin(), r"AppData\Local\lbry")
self.assertEqual(c.data_dir, os.path.join(prefix, 'lbrynet'))
self.assertEqual(c.wallet_dir, os.path.join(prefix, 'lbryum'))
self.assertEqual(c.download_dir, os.path.join(r"C:\Users", os.getlogin(), "Downloads"))
self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml'))
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log'))
@unittest.skipIf('linux' not in sys.platform, 'skipping linux only test')
def test_linux_defaults(self):
c = Config()