Compare commits

...
Author SHA1 Message Date
Lex Berezhny 7582c221d1 v0.102.0
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
2021-08-17 14:16:17 -04:00
Lex Berezhny c109895848 Merge pull request #3399 from lbryio/better-error-logging
Less verbose error logs, only log tracebacks for errors not defined in `lbry.error`
2021-08-17 14:14:27 -04:00
Jack Robison eccedada40 add TODOs for errors raised that aren't defined in lbry.error 2021-08-17 12:31:03 -04:00
Jack Robison 25d54accf8 return api errors from wallet_add and wallet_create 2021-08-17 12:30:17 -04:00
Jack Robison d07685f0e9 only log tracebacks for api errors not defined in lbry.error 2021-08-17 11:30:58 -04:00
Jack Robison 2445c00c7e raise WalletNotLoadedError in get_wallet_or_error instead of ValueError 2021-08-17 11:30:58 -04:00
Lex Berezhny 4c1d3ef514 Merge pull request #3398 from lbryio/clean_blobs_after_delay
clean blobs after waiting interval instead of immediately on startup
2021-08-17 10:44:34 -04:00
Lex Berezhny 4614c7d4c2 clean blobs after waiting interval instead of immediately on startup 2021-08-17 09:52:44 -04:00
Lex Berezhny bbf1ef0dc3 Merge pull request #3378 from lbryio/disk_management
ability to limit disk spaced used for blobs via `blob_storage_limit` setting (oldest blobs are deleted when disk space limit is reached)
2021-08-16 17:41:05 -04:00
Lex Berezhny 3433c9e708 return number of files deleted 2021-08-16 17:03:40 -04:00
Lex Berezhny 2cd5d75a2e return true/false if clean was performed 2021-08-16 17:02:13 -04:00
Lex Berezhny 2535b8adef fix disk space unit test 2021-08-16 14:54:17 -04:00
Lex Berezhny 4edab7bb7f fix sorting by DirEntry error 2021-08-16 14:41:16 -04:00
Lex Berezhny fd8658e317 test component unit test 2021-08-16 14:35:32 -04:00
Lex Berezhny 51d21d8c86 working disk cleanup 2021-08-16 14:15:12 -04:00
Lex Berezhny b4c3307cdf fixed tests 2021-08-13 10:32:46 -04:00
Lex Berezhny 4e8d10cb44 disk space manager and status API 2021-08-13 10:32:46 -04:00
Lex Berezhny e96875a425 workflow syntax 2021-08-13 10:32:46 -04:00
Lex Berezhny 5ab0035348 run tests on windows and mac 2021-08-13 10:32:46 -04:00
Lex Berezhny 4ddff96b1e Merge pull request #3395 from lbryio/libtorrent_optional
make libtorrent optional and skip test which depends on it
2021-08-13 10:31:06 -04:00
Lex Berezhny a08d84c1df make libtorrent optional and skip test which depends on it 2021-08-13 10:07:06 -04:00
14 changed files with 309 additions and 17 deletions

No files matched your search

+12 -4
View File
@@ -31,8 +31,8 @@ jobs:
matrix:
os:
- ubuntu-latest
#- macos-latest
#- windows-latest
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
@@ -49,10 +49,18 @@ jobs:
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: pip install --user --upgrade pip wheel
- run: pip install -e .[torrent,test]
- env:
- if: startsWith(runner.os, 'linux')
run: pip install -e .[torrent,test]
- if: startsWith(runner.os, 'linux')
env:
HOME: /tmp
run: make test-unit-coverage
- if: startsWith(runner.os, 'linux') != true
run: pip install -e .[test]
- if: startsWith(runner.os, 'linux') != true
env:
HOME: /tmp
run: python -m unittest tests/unit/test_conf.py
tests-integration:
name: "tests / integration"
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.101.1"
__version__ = "0.102.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+64
View File
@@ -0,0 +1,64 @@
import os
import asyncio
import logging
log = logging.getLogger(__name__)
class DiskSpaceManager:
def __init__(self, config, cleaning_interval=30 * 60):
self.config = config
self.cleaning_interval = cleaning_interval
self.running = False
self.task = None
@property
def space_used_bytes(self):
used = 0
data_dir = os.path.join(self.config.data_dir, 'blobfiles')
for item in os.scandir(data_dir):
if item.is_file:
used += item.stat().st_size
return used
@property
def space_used_mb(self):
return int(self.space_used_bytes/1024.0/1024.0)
def clean(self):
if not self.config.blob_storage_limit:
return 0
used = 0
files = []
data_dir = os.path.join(self.config.data_dir, 'blobfiles')
for file in os.scandir(data_dir):
if file.is_file:
file_stats = file.stat()
used += file_stats.st_size
files.append((file_stats.st_mtime, file_stats.st_size, file.path))
files.sort()
available = (self.config.blob_storage_limit*1024*1024) - used
cleaned = 0
for _, file_size, file in files:
available += file_size
if available > 0:
break
os.remove(file)
cleaned += 1
return cleaned
async def cleaning_loop(self):
while self.running:
await asyncio.sleep(self.cleaning_interval)
await asyncio.get_event_loop().run_in_executor(None, self.clean)
async def start(self):
self.running = True
self.task = asyncio.create_task(self.cleaning_loop())
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
async def stop(self):
if self.running:
self.running = False
self.task.cancel()
+1
View File
@@ -634,6 +634,7 @@ class Config(CLIConfig):
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32
+4
View File
@@ -61,6 +61,10 @@ Code | Name | Message
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
436 | WalletNotFound | Wallet not found at {wallet_path}.
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
**5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob.
+28
View File
@@ -271,6 +271,34 @@ class ServerPaymentFeeAboveMaxAllowedError(WalletError):
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
class WalletNotLoadedError(WalletError):
def __init__(self, wallet_id):
self.wallet_id = wallet_id
super().__init__(f"Wallet {wallet_id} is not loaded.")
class WalletAlreadyLoadedError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet {wallet_path} is already loaded.")
class WalletNotFoundError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet not found at {wallet_path}.")
class WalletAlreadyExistsError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
class BlobError(BaseError):
"""
**Blobs**
+26
View File
@@ -15,6 +15,7 @@ from lbry.dht.node import Node
from lbry.dht.peer import is_valid_public_ipv4
from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.blob.blob_manager import BlobManager
from lbry.blob.disk_space_manager import DiskSpaceManager
from lbry.blob_exchange.server import BlobServer
from lbry.stream.stream_manager import StreamManager
from lbry.file.file_manager import FileManager
@@ -40,6 +41,7 @@ WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
FILE_MANAGER_COMPONENT = "file_manager"
DISK_SPACE_COMPONENT = "disk_space"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
@@ -375,6 +377,30 @@ class FileManagerComponent(Component):
self.file_manager.stop()
class DiskSpaceComponent(Component):
component_name = DISK_SPACE_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.disk_space_manager = DiskSpaceManager(self.conf)
@property
def component(self) -> typing.Optional[DiskSpaceManager]:
return self.disk_space_manager
async def get_status(self):
return {
'space_used': str(self.disk_space_manager.space_used_mb),
'running': self.disk_space_manager.running,
}
async def start(self):
await self.disk_space_manager.start()
async def stop(self):
await self.disk_space_manager.stop()
class TorrentComponent(Component):
component_name = LIBTORRENT_COMPONENT
+93 -9
View File
@@ -36,12 +36,12 @@ from lbry.blob_exchange.downloader import download_blob
from lbry.dht.peer import make_kademlia_peer
from lbry.error import (
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
CommandDoesNotExistError
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError
)
from lbry.extras import system_info
from lbry.extras.daemon import analytics
from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT
from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT
from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT
from lbry.extras.daemon.componentmanager import RequiredCondition
from lbry.extras.daemon.componentmanager import ComponentManager
@@ -56,7 +56,7 @@ from lbry.schema.url import URL
if typing.TYPE_CHECKING:
from lbry.blob.blob_manager import BlobManager
from lbry.dht.node import Node
from lbry.extras.daemon.components import UPnPComponent
from lbry.extras.daemon.components import UPnPComponent, DiskSpaceManager
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager, Ledger
@@ -375,6 +375,10 @@ class Daemon(metaclass=JSONRPCServerType):
def blob_manager(self) -> typing.Optional['BlobManager']:
return self.component_manager.get_component(BLOB_COMPONENT)
@property
def disk_space_manager(self) -> typing.Optional['DiskSpaceManager']:
return self.component_manager.get_component(DISK_SPACE_COMPONENT)
@property
def upnp(self) -> typing.Optional['UPnPComponent']:
return self.component_manager.get_component(UPNP_COMPONENT)
@@ -698,7 +702,10 @@ class Daemon(metaclass=JSONRPCServerType):
raise
except Exception as e: # pylint: disable=broad-except
self.failed_request_metric.labels(method=function_name).inc()
log.exception("error handling api request")
if not isinstance(e, BaseError):
log.exception("error handling api request")
else:
log.error("error handling api request: %s", e)
return JSONRPCError.create_command_exception(
command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc()
)
@@ -1103,6 +1110,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not stream:
raise DownloadSDTimeoutError(uri)
except Exception as e:
# TODO: use error from lbry.error
log.warning("Error downloading %s: %s", uri, str(e))
return {"error": str(e)}
return stream
@@ -1269,9 +1277,9 @@ class Daemon(metaclass=JSONRPCServerType):
wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id)
for wallet in self.wallet_manager.wallets:
if wallet.id == wallet_id:
raise Exception(f"Wallet at path '{wallet_path}' already exists and is loaded.")
raise WalletAlreadyLoadedError(wallet_path)
if os.path.exists(wallet_path):
raise Exception(f"Wallet at path '{wallet_path}' already exists, use 'wallet_add' to load wallet.")
raise WalletAlreadyExistsError(wallet_path)
wallet = self.wallet_manager.import_wallet(wallet_path)
if not wallet.accounts and create_account:
@@ -1304,9 +1312,9 @@ class Daemon(metaclass=JSONRPCServerType):
wallet_path = os.path.join(self.conf.wallet_dir, 'wallets', wallet_id)
for wallet in self.wallet_manager.wallets:
if wallet.id == wallet_id:
raise Exception(f"Wallet at path '{wallet_path}' is already loaded.")
raise WalletAlreadyLoadedError(wallet_path)
if not os.path.exists(wallet_path):
raise Exception(f"Wallet at path '{wallet_path}' was not found.")
raise WalletNotFoundError(wallet_path)
wallet = self.wallet_manager.import_wallet(wallet_path)
if self.ledger.network.is_connected:
for account in wallet.accounts:
@@ -1491,7 +1499,7 @@ class Daemon(metaclass=JSONRPCServerType):
)
)
else:
raise ValueError(f"Unsupported address: '{address}'")
raise ValueError(f"Unsupported address: '{address}'") # TODO: use error from lbry.error
tx = await Transaction.create(
[], outputs, accounts, account
@@ -1769,8 +1777,10 @@ class Daemon(metaclass=JSONRPCServerType):
from_account = wallet.get_account_or_default(from_account)
amount = self.get_dewies_or_error('amount', amount) if amount else None
if not isinstance(outputs, int):
# TODO: use error from lbry.error
raise ValueError("--outputs must be an integer.")
if everything and outputs > 1:
# TODO: use error from lbry.error
raise ValueError("Using --everything along with --outputs is not supported.")
return from_account.fund(
to_account=to_account, amount=amount, everything=everything,
@@ -2041,10 +2051,12 @@ class Daemon(metaclass=JSONRPCServerType):
"""
if status not in ['start', 'stop']:
# TODO: use error from lbry.error
raise Exception('Status must be "start" or "stop".')
streams = self.file_manager.get_filtered(**kwargs)
if not streams:
# TODO: use error from lbry.error
raise Exception(f'Unable to find a file for {kwargs}')
stream = streams[0]
if status == 'start' and not stream.running:
@@ -2227,20 +2239,25 @@ class Daemon(metaclass=JSONRPCServerType):
if claim_id:
txo = await self.ledger.get_claim_by_claim_id(accounts, claim_id, include_purchase_receipt=True)
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find claim with claim_id '{claim_id}'. ")
elif url:
txo = (await self.ledger.resolve(accounts, [url], include_purchase_receipt=True))[url]
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find claim with url '{url}'. ")
else:
# TODO: use error from lbry.error
raise Exception(f"Missing argument claim_id or url. ")
if not allow_duplicate_purchase and txo.purchase_receipt:
# TODO: use error from lbry.error
raise Exception(
f"You already have a purchase for claim_id '{claim_id}'. "
f"Use --allow-duplicate-purchase flag to override."
)
claim = txo.claim
if not claim.is_stream or not claim.stream.has_fee:
# TODO: use error from lbry.error
raise Exception(f"Claim '{claim_id}' does not have a purchase price.")
tx = await self.wallet_manager.create_purchase_transaction(
accounts, txo, self.exchange_rate_manager, override_max_key_fee
@@ -2482,6 +2499,7 @@ class Daemon(metaclass=JSONRPCServerType):
if "claim_ids" in kwargs and not kwargs["claim_ids"]:
kwargs.pop("claim_ids")
if {'claim_id', 'claim_ids'}.issubset(kwargs):
# TODO: use error from lbry.error
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
if kwargs.pop('valid_channel_signature', False):
kwargs['signature_valid'] = 1
@@ -2601,6 +2619,7 @@ class Daemon(metaclass=JSONRPCServerType):
existing_channels = await self.ledger.get_channels(accounts=wallet.accounts, claim_name=name)
if len(existing_channels) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a channel under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
@@ -2733,11 +2752,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(existing_channels) != 1:
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
# TODO: use error from lbry.error
raise Exception(
f"Can't find the channel '{claim_id}' in account(s) {account_ids}."
)
old_txo = existing_channels[0]
if not old_txo.claim.is_channel:
# TODO: use error from lbry.error
raise Exception(
f"A claim with id '{claim_id}' was found but it is not a channel."
)
@@ -2860,9 +2881,11 @@ class Daemon(metaclass=JSONRPCServerType):
wallet=wallet, accounts=accounts, claim_id=claim_id
)
else:
# TODO: use error from lbry.error
raise Exception('Must specify claim_id, or txid and nout')
if not claims:
# TODO: use error from lbry.error
raise Exception('No claim found for the specified claim_id or txid:nout')
tx = await Transaction.create(
@@ -2931,6 +2954,7 @@ class Daemon(metaclass=JSONRPCServerType):
address = channel.get_address(self.ledger)
public_key = await self.ledger.get_public_key_for_address(wallet, address)
if not public_key:
# TODO: use error from lbry.error
raise Exception("Can't find public key for address holding the channel.")
export = {
'name': channel.claim_name,
@@ -2992,6 +3016,7 @@ class Daemon(metaclass=JSONRPCServerType):
await self.ledger._update_tasks.done.wait()
# Case 3: the holding address has changed and we can't create or find an account for it
else:
# TODO: use error from lbry.error
raise Exception(
"Channel owning account has changed since the channel was exported and "
"it is not an account to which you have access."
@@ -3117,11 +3142,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(claims) == 0:
if 'bid' not in kwargs:
# TODO: use error from lbry.error
raise Exception("'bid' is a required argument for new publishes.")
return await self.jsonrpc_stream_create(name, **kwargs)
elif len(claims) == 1:
assert claims[0].claim.is_stream, f"Claim at name '{name}' is not a stream claim."
return await self.jsonrpc_stream_update(claims[0].claim_id, replace=True, **kwargs)
# TODO: use error from lbry.error
raise Exception(
f"There are {len(claims)} claims for '{name}', please use 'stream update' command "
f"to update a specific stream claim."
@@ -3173,11 +3200,13 @@ class Daemon(metaclass=JSONRPCServerType):
claims = await account.get_claims(claim_name=name)
if len(claims) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a stream claim published under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
)
if not VALID_FULL_CLAIM_ID.fullmatch(claim_id):
# TODO: use error from lbry.error
raise Exception('Invalid claim id. It is expected to be a 40 characters long hexadecimal string.')
claim = Claim()
@@ -3321,6 +3350,7 @@ class Daemon(metaclass=JSONRPCServerType):
claims = await account.get_claims(claim_name=name)
if len(claims) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a stream claim published under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
@@ -3504,11 +3534,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(existing_claims) != 1:
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
# TODO: use error from lbry.error
raise Exception(
f"Can't find the stream '{claim_id}' in account(s) {account_ids}."
)
old_txo = existing_claims[0]
if not old_txo.claim.is_stream:
# TODO: use error from lbry.error
raise Exception(
f"A claim with id '{claim_id}' was found but it is not a stream claim."
)
@@ -3636,9 +3668,11 @@ class Daemon(metaclass=JSONRPCServerType):
wallet=wallet, accounts=accounts, claim_id=claim_id
)
else:
# TODO: use error from lbry.error
raise Exception('Must specify claim_id, or txid and nout')
if not claims:
# TODO: use error from lbry.error
raise Exception('No claim found for the specified claim_id or txid:nout')
tx = await Transaction.create(
@@ -3800,6 +3834,7 @@ class Daemon(metaclass=JSONRPCServerType):
existing_collections = await self.ledger.get_collections(accounts=wallet.accounts, claim_name=name)
if len(existing_collections) > 0:
if not allow_duplicate_name:
# TODO: use error from lbry.error
raise Exception(
f"You already have a collection under the name '{name}'. "
f"Use --allow-duplicate-name flag to override."
@@ -3923,11 +3958,13 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(existing_collections) != 1:
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
# TODO: use error from lbry.error
raise Exception(
f"Can't find the collection '{claim_id}' in account(s) {account_ids}."
)
old_txo = existing_collections[0]
if not old_txo.claim.is_collection:
# TODO: use error from lbry.error
raise Exception(
f"A claim with id '{claim_id}' was found but it is not a collection."
)
@@ -4053,12 +4090,15 @@ class Daemon(metaclass=JSONRPCServerType):
if claim_id:
txo = await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id)
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find collection with claim_id '{claim_id}'. ")
elif url:
txo = (await self.ledger.resolve(wallet.accounts, [url]))[url]
if not isinstance(txo, Output) or not txo.is_claim:
# TODO: use error from lbry.error
raise Exception(f"Could not find collection with url '{url}'. ")
else:
# TODO: use error from lbry.error
raise Exception(f"Missing argument claim_id or url. ")
page_num, page_size = abs(page), min(abs(page_size), 50)
@@ -4233,9 +4273,11 @@ class Daemon(metaclass=JSONRPCServerType):
wallet=wallet, accounts=accounts, claim_id=claim_id
)
else:
# TODO: use error from lbry.error
raise Exception('Must specify claim_id, or txid and nout')
if not supports:
# TODO: use error from lbry.error
raise Exception('No supports found for the specified claim_id or txid:nout')
if keep is not None:
@@ -4470,6 +4512,7 @@ class Daemon(metaclass=JSONRPCServerType):
elif order_by in ('height', 'amount', 'none'):
constraints['order_by'] = order_by
else:
# TODO: use error from lbry.error
raise ValueError(f"'{order_by}' is not a valid --order_by value.")
self._constrain_txo_from_kwargs(constraints, **kwargs)
return paginate_rows(claims, None if no_totals else claim_count, page, page_size, **constraints)
@@ -4776,10 +4819,12 @@ class Daemon(metaclass=JSONRPCServerType):
"""
if not is_valid_blobhash(blob_hash):
# TODO: use error from lbry.error
raise Exception("invalid blob hash")
if search_bottom_out_limit is not None:
search_bottom_out_limit = int(search_bottom_out_limit)
if search_bottom_out_limit <= 0:
# TODO: use error from lbry.error
raise Exception("invalid bottom out limit")
else:
search_bottom_out_limit = 4
@@ -4823,12 +4868,14 @@ class Daemon(metaclass=JSONRPCServerType):
blob_hashes.append(blob_hash)
elif stream_hash or sd_hash:
if sd_hash and stream_hash:
# TODO: use error from lbry.error
raise Exception("either the sd hash or the stream hash should be provided, not both")
if sd_hash:
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
blobs = await self.storage.get_blobs_for_stream(stream_hash, only_completed=True)
blob_hashes.extend(blob.blob_hash for blob in blobs if blob.blob_hash is not None)
else:
# TODO: use error from lbry.error
raise Exception('single argument must be specified')
await self.storage.should_single_announce_blobs(blob_hashes, immediate=True)
return True
@@ -4916,6 +4963,22 @@ class Daemon(metaclass=JSONRPCServerType):
raise NotImplementedError()
@requires(DISK_SPACE_COMPONENT)
async def jsonrpc_blob_clean(self):
"""
Deletes blobs to cleanup disk space
Usage:
blob_clean
Options:
None
Returns:
(bool) true if successful
"""
return self.disk_space_manager.clean()
@requires(FILE_MANAGER_COMPONENT)
async def jsonrpc_file_reflect(self, **kwargs):
"""
@@ -5079,6 +5142,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
if not tracemalloc.is_tracing():
# TODO: use error from lbry.error
raise Exception("Enable tracemalloc first! See 'tracemalloc set' command.")
stats = tracemalloc.take_snapshot().filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
@@ -5288,6 +5352,7 @@ class Daemon(metaclass=JSONRPCServerType):
comment_id=comment_id
)
if 'error' in channel:
# TODO: use error from lbry.error
raise ValueError(channel['error'])
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
@@ -5544,51 +5609,64 @@ class Daemon(metaclass=JSONRPCServerType):
allow_script_address and self.ledger.is_script_address(address)
)
except:
# TODO: use error from lbry.error
raise Exception(f"'{address}' is not a valid address")
@staticmethod
def valid_stream_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception('Stream name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
# TODO: use error from lbry.error
raise Exception(
"Stream names cannot start with '@' symbol. This is reserved for channels claims."
)
if not parsed.has_stream or parsed.stream.name != name:
# TODO: use error from lbry.error
raise Exception('Stream name has invalid characters.')
except (TypeError, ValueError):
# TODO: use error from lbry.error
raise Exception("Invalid stream name.")
@staticmethod
def valid_collection_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception('Collection name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
# TODO: use error from lbry.error
raise Exception(
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
)
if not parsed.has_stream or parsed.stream.name != name:
# TODO: use error from lbry.error
raise Exception('Collection name has invalid characters.')
except (TypeError, ValueError):
# TODO: use error from lbry.error
raise Exception("Invalid collection name.")
@staticmethod
def valid_channel_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception(
"Channel name cannot be blank."
)
parsed = URL.parse(name)
if not parsed.has_channel:
# TODO: use error from lbry.error
raise Exception("Channel names must start with '@' symbol.")
if parsed.channel.name != name:
# TODO: use error from lbry.error
raise Exception("Channel name has invalid character")
except (TypeError, ValueError):
# TODO: use error from lbry.error
raise Exception("Invalid channel name.")
def get_fee_address(self, kwargs: dict, claim_address: str) -> str:
@@ -5620,6 +5698,7 @@ class Daemon(metaclass=JSONRPCServerType):
elif channel_name:
key, value = 'name', channel_name
else:
# TODO: use error from lbry.error
raise ValueError("Couldn't find channel because a channel_id or channel_name was not provided.")
channels = await self.ledger.get_channels(
wallet=wallet, accounts=wallet.get_accounts_or_all(account_ids),
@@ -5627,13 +5706,16 @@ class Daemon(metaclass=JSONRPCServerType):
)
if len(channels) == 1:
if for_signing and not channels[0].has_private_key:
# TODO: use error from lbry.error
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
return channels[0]
elif len(channels) > 1:
# TODO: use error from lbry.error
raise ValueError(
f"Multiple channels found with channel_{key} '{value}', "
f"pass a channel_id to narrow it down."
)
# TODO: use error from lbry.error
raise ValueError(f"Couldn't find channel with channel_{key} '{value}'.")
@staticmethod
@@ -5641,9 +5723,11 @@ class Daemon(metaclass=JSONRPCServerType):
try:
dewies = lbc_to_dewies(lbc)
if positive_value and dewies <= 0:
# TODO: use error from lbry.error
raise ValueError(f"'{argument}' value must be greater than 0.0")
return dewies
except ValueError as e:
# TODO: use error from lbry.error
raise ValueError(f"Invalid value for '{argument}': {e.args[0]}")
async def resolve(self, accounts, urls, **kwargs):
+6
View File
@@ -607,6 +607,12 @@ class CommandTestCase(IntegrationTestCase):
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
return self.sout(txs)
async def blob_clean(self):
return await self.out(self.daemon.jsonrpc_blob_clean())
async def status(self):
return await self.out(self.daemon.jsonrpc_status())
async def resolve(self, uri, **kwargs):
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
+2 -2
View File
@@ -7,7 +7,7 @@ from binascii import unhexlify
from decimal import Decimal
from typing import List, Type, MutableSequence, MutableMapping, Optional
from lbry.error import KeyFeeAboveMaxAllowedError
from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
from lbry.conf import Config, NOT_SET
from .dewies import dewies_to_lbc
@@ -95,7 +95,7 @@ class WalletManager:
for wallet in self.wallets:
if wallet.id == wallet_id:
return wallet
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
raise WalletNotLoadedError(wallet_id)
@staticmethod
def get_balance(wallet):
@@ -1,10 +1,11 @@
from unittest import skipIf
import asyncio
import os
from binascii import hexlify
from lbry.schema import Claim
from lbry.testcase import CommandTestCase
from lbry.torrent.session import TorrentSession
from lbry.extras.daemon.components import TorrentSession
from lbry.wallet import Transaction
@@ -40,6 +41,7 @@ class FileCommands(CommandTestCase):
self.client_session.wait_start = False # fixme: this is super slow on tests
return tx, btih
@skipIf(TorrentSession is None, "libtorrent not installed")
async def test_download_torrent(self):
tx, btih = await self.initialize_torrent()
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
@@ -509,3 +511,20 @@ class FileCommands(CommandTestCase):
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
class DiskSpaceManagement(CommandTestCase):
async def test_file_management(self):
status = await self.status()
self.assertIn('disk_space', status)
self.assertEqual('0', status['disk_space']['space_used'])
self.assertEqual(True, status['disk_space']['running'])
await self.stream_create('foo1', '0.01', data=('0' * 3 * 1024 * 1024).encode())
await self.stream_create('foo2', '0.01', data=('0' * 2 * 1024 * 1024).encode())
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
await self.blob_clean()
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
self.daemon.conf.blob_storage_limit = 3
await self.blob_clean()
self.assertEqual('3', (await self.status())['disk_space']['space_used'])
@@ -0,0 +1,30 @@
import os
import unittest
import tempfile
import lbry.wallet
from lbry.conf import Config
from lbry.blob.disk_space_manager import DiskSpaceManager
class ConfigurationTests(unittest.TestCase):
def test_space_management(self):
with tempfile.TemporaryDirectory() as temp_dir:
os.mkdir(os.path.join(temp_dir, 'blobfiles'))
config = Config(
blob_storage_limit=5,
data_dir=temp_dir,
wallet_dir=temp_dir,
config=os.path.join(temp_dir, 'settings.yml'),
)
dsm = DiskSpaceManager(config)
self.assertEqual(0, dsm.space_used_mb)
for file_no in range(10):
with open(os.path.join(config.data_dir, 'blobfiles', f'3mb-{file_no}'), 'w') as blob:
blob.write('0' * 1 * 1024 * 1024)
self.assertEqual(10, dsm.space_used_mb)
self.assertTrue(dsm.clean())
self.assertEqual(5, dsm.space_used_mb)
self.assertFalse(dsm.clean())
@@ -15,6 +15,7 @@ class TestComponentManager(AsyncioTestCase):
self.default_components_sort = [
[
components.DatabaseComponent,
components.DiskSpaceComponent,
components.ExchangeRateManagerComponent,
components.TorrentComponent,
components.UPnPComponent
+21
View File
@@ -21,6 +21,27 @@ class TestConfig(BaseConfig):
class ConfigurationTests(unittest.TestCase):
@unittest.skipIf('darwin' not in sys.platform, 'skipping mac only test')
def test_mac_defaults(self):
c = Config()
self.assertEqual(c.data_dir, os.path.expanduser("~/Library/Application Support/LBRY"))
self.assertEqual(c.wallet_dir, os.path.expanduser('~/.lbryum'))
self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads'))
self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml'))
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log'))
@unittest.skipIf('win32' not in sys.platform, 'skipping windows only test')
def test_windows_defaults(self):
c = Config()
prefix = os.path.join(r"C:\Users", os.getlogin(), r"AppData\Local\lbry")
self.assertEqual(c.data_dir, os.path.join(prefix, 'lbrynet'))
self.assertEqual(c.wallet_dir, os.path.join(prefix, 'lbryum'))
self.assertEqual(c.download_dir, os.path.join(r"C:\Users", os.getlogin(), "Downloads"))
self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml'))
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log'))
@unittest.skipIf('linux' not in sys.platform, 'skipping linux only test')
def test_linux_defaults(self):
c = Config()