Compare commits

..
Author SHA1 Message Date
Lex Berezhny 84908ec8ec v0.105.0
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (blockchain_legacy_search) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
2021-10-05 11:29:39 -04:00
Victor Shyba dade49743b fix file reflect and add test 2021-10-04 19:26:05 -03:00
Lex Berezhny f29bf35c2a Merge pull request #3438 from lbryio/disk_space_metrics
metrics reported now include disk space consumed by blobs and what the disk usage limit, if any, is set to
2021-10-03 20:01:50 -04:00
Lex Berezhny dfa6701c43 disk space metrics 2021-10-03 19:33:18 -04:00
Victor Shyba 763ca69a73 dht: use bytes hex/fromhex instead of binascii 2021-09-30 13:26:33 -03:00
Victor Shyba 6bf3b152bf add grin to dht known list 2021-09-30 13:26:33 -03:00
Victor Shyba aa19f85996 add madiator to known dht nodes 2021-09-30 13:26:33 -03:00
Victor Shyba 156d89567e add option to set bootstrap_node 2021-09-30 13:26:33 -03:00
Victor Shyba ecc71baf61 add dockerfile for dht node 2021-09-30 13:26:33 -03:00
Victor Shyba 90c743d963 configure where to save peers 2021-09-30 13:26:33 -03:00
Victor Shyba b926293fa7 define arg types 2021-09-30 13:26:33 -03:00
Victor Shyba 71a19191f8 add dht seed node script 2021-09-30 13:26:33 -03:00
Victor Shyba 38a0f20a33 fix conflict with imported function 2021-09-30 13:24:17 -03:00
Victor Shyba c35192108c errors for empyt and misssing file on publish 2021-09-30 13:24:17 -03:00
Victor Shyba 245b564f13 generalize stream empty to argument empty 2021-09-30 13:24:17 -03:00
Victor Shyba 0d8d1ea4f3 empty stream name error for user input 2021-09-30 13:24:17 -03:00
Victor Shyba 27a427a363 error for missing channel private key 2021-09-30 13:24:17 -03:00
Victor Shyba 2ff028a694 error for already purchased claims 2021-09-30 13:24:17 -03:00
14 changed files with 103 additions and 54 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.104.0"
__version__ = "0.105.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+8 -3
View File
@@ -6,13 +6,14 @@ log = logging.getLogger(__name__)
class DiskSpaceManager:
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60):
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None):
self.config = config
self.db = db
self.blob_manager = blob_manager
self.cleaning_interval = cleaning_interval
self.running = False
self.task = None
self.analytics = analytics
async def get_space_used_bytes(self):
return await self.db.get_stored_blob_disk_usage()
@@ -21,10 +22,14 @@ class DiskSpaceManager:
return int(await self.get_space_used_bytes()/1024.0/1024.0)
async def clean(self):
if not self.config.blob_storage_limit:
space_used_bytes = await self.get_space_used_bytes()
storage_limit = self.config.blob_storage_limit*1024*1024 if self.config.blob_storage_limit else None
if self.analytics:
asyncio.create_task(self.analytics.send_disk_space_used(space_used_bytes, storage_limit))
if not storage_limit:
return 0
delete = []
available = (self.config.blob_storage_limit*1024*1024) - await self.get_space_used_bytes()
available = storage_limit - space_used_bytes
if available > 0:
return 0
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False):
+3 -9
View File
@@ -18,7 +18,7 @@ MAX_REQUEST_SIZE = 1200
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
idle_timeout: float = 30.0, transfer_timeout: float = 60.0, blob_callback=None):
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
self.loop = loop
self.blob_manager = blob_manager
self.idle_timeout = idle_timeout
@@ -32,7 +32,6 @@ class BlobServerProtocol(asyncio.Protocol):
self.started_transfer = asyncio.Event(loop=self.loop)
self.transfer_finished = asyncio.Event(loop=self.loop)
self.close_on_idle_task: typing.Optional[asyncio.Task] = None
self.blob_handling_callback: typing.Optional[typing.Callable] = blob_callback
async def close_on_idle(self):
while self.transport:
@@ -93,9 +92,6 @@ class BlobServerProtocol(asyncio.Protocol):
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if self.blob_handling_callback:
await self.blob_handling_callback(blob)
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
@@ -156,7 +152,7 @@ class BlobServerProtocol(asyncio.Protocol):
class BlobServer:
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
idle_timeout: float = 30.0, transfer_timeout: float = 60.0, blob_callback=None):
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
@@ -165,7 +161,6 @@ class BlobServer:
self.idle_timeout = idle_timeout
self.transfer_timeout = transfer_timeout
self.server_protocol_class = BlobServerProtocol
self.blob_handling_callback: typing.Optional[typing.Callable] = blob_callback
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -174,8 +169,7 @@ class BlobServer:
async def _start_server():
server = await self.loop.create_server(
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
self.idle_timeout, self.transfer_timeout,
blob_callback=self.blob_handling_callback),
self.idle_timeout, self.transfer_timeout),
interface, port
)
self.started_listening.set()
+2 -1
View File
@@ -96,7 +96,8 @@ class KademliaRPC:
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
]
# if we don't have k storing peers to return and we have this hash locally, include our contact information
peers.append(self.compact_address())
if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address())
if not peers:
response[PAGE_KEY] = 0
else:
+5
View File
@@ -35,6 +35,9 @@ Code | Name | Message
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
114 | InputStringIsBlank | {argument} cannot be blank.
115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
116 | MissingPublishedFile | File does not exist: {file_path}
**2xx** | Configuration | Configuration errors.
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
@@ -52,6 +55,7 @@ Code | Name | Message
405 | ChannelKeyNotFound | Channel signing key not found.
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
407 | DataDownload | Failed to download blob. *generic*
408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
@@ -59,6 +63,7 @@ Code | Name | Message
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
+39
View File
@@ -84,6 +84,27 @@ class ConflictingInputValueError(InputValueError):
super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
class InputStringIsBlankError(InputValueError):
def __init__(self, argument):
self.argument = argument
super().__init__(f"{argument} cannot be blank.")
class EmptyPublishedFileError(InputValueError):
def __init__(self, file_path):
self.file_path = file_path
super().__init__(f"Cannot publish empty file: {file_path}")
class MissingPublishedFileError(InputValueError):
def __init__(self, file_path):
self.file_path = file_path
super().__init__(f"File does not exist: {file_path}")
class ConfigurationError(BaseError):
"""
Configuration errors.
@@ -207,6 +228,14 @@ class DataDownloadError(WalletError):
super().__init__("Failed to download blob. *generic*")
class PrivateKeyNotFoundError(WalletError):
def __init__(self, key, value):
self.key = key
self.value = value
super().__init__(f"Couldn't find private key for {key} '{value}'.")
class ResolveError(WalletError):
def __init__(self, url):
@@ -258,6 +287,16 @@ class TooManyClaimSearchParametersError(WalletError):
super().__init__(f"{key} cant have more than {limit} items.")
class AlreadyPurchasedError(WalletError):
"""
allow-duplicate-purchase flag to override.
"""
def __init__(self, claim_id_hex):
self.claim_id_hex = claim_id_hex
super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
+9
View File
@@ -18,6 +18,7 @@ DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
DISK_SPACE = 'Disk Space'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
@@ -169,6 +170,14 @@ class AnalyticsManager:
})
)
async def send_disk_space_used(self, storage_used, storage_limit):
await self.track(
self._event(DISK_SPACE, {
'used': storage_used,
'limit': storage_limit,
})
)
async def send_server_startup(self):
await self.track(self._event(SERVER_STARTUP))
+4 -1
View File
@@ -400,7 +400,10 @@ class DiskSpaceComponent(Component):
async def start(self):
db = self.component_manager.get_component(DATABASE_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
self.disk_space_manager = DiskSpaceManager(self.conf, db, blob_manager)
self.disk_space_manager = DiskSpaceManager(
self.conf, db, blob_manager,
analytics=self.component_manager.analytics_manager
)
await self.disk_space_manager.start()
async def stop(self):
+6 -11
View File
@@ -38,7 +38,7 @@ from lbry.dht.peer import make_kademlia_peer
from lbry.error import (
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError,
ConflictingInputValueError
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError
)
from lbry.extras import system_info
from lbry.extras.daemon import analytics
@@ -2295,11 +2295,7 @@ class Daemon(metaclass=JSONRPCServerType):
# TODO: use error from lbry.error
raise Exception("Missing argument claim_id or url.")
if not allow_duplicate_purchase and txo.purchase_receipt:
# TODO: use error from lbry.error
raise Exception(
f"You already have a purchase for claim_id '{claim_id}'. "
f"Use --allow-duplicate-purchase flag to override."
)
raise AlreadyPurchasedError(claim_id)
claim = txo.claim
if not claim.is_stream or not claim.stream.has_fee:
# TODO: use error from lbry.error
@@ -5072,8 +5068,8 @@ class Daemon(metaclass=JSONRPCServerType):
else:
server, port = random.choice(self.conf.reflector_servers)
reflected = await asyncio.gather(*[
self.file_manager['stream'].reflect_stream(stream, server, port)
for stream in self.file_manager.get_filtered_streams(**kwargs)
self.file_manager.source_managers['stream'].reflect_stream(stream, server, port)
for stream in self.file_manager.get_filtered(**kwargs)
])
total = []
for reflected_for_stream in reflected:
@@ -5246,8 +5242,7 @@ class Daemon(metaclass=JSONRPCServerType):
def valid_stream_name_or_error(name: str):
try:
if not name:
# TODO: use error from lbry.error
raise Exception('Stream name cannot be blank.')
raise InputStringIsBlankError('Stream name')
parsed = URL.parse(name)
if parsed.has_channel:
# TODO: use error from lbry.error
@@ -5337,7 +5332,7 @@ class Daemon(metaclass=JSONRPCServerType):
if len(channels) == 1:
if for_signing and not channels[0].has_private_key:
# TODO: use error from lbry.error
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
raise PrivateKeyNotFoundError(key, value)
return channels[0]
elif len(channels) > 1:
# TODO: use error from lbry.error
+3 -2
View File
@@ -10,6 +10,7 @@ from google.protobuf.json_format import MessageToDict
from lbry.crypto.base58 import Base58
from lbry.constants import COIN
from lbry.error import MissingPublishedFileError, EmptyPublishedFileError
from lbry.schema.mime_types import guess_media_type
from lbry.schema.base import Metadata, BaseMessageList
@@ -139,10 +140,10 @@ class Source(Metadata):
self.name = os.path.basename(file_path)
self.media_type, stream_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}")
raise MissingPublishedFileError(file_path)
self.size = os.path.getsize(file_path)
if self.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}")
raise EmptyPublishedFileError(file_path)
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
return stream_type
+1
View File
@@ -228,6 +228,7 @@ class StreamManager(SourceManager):
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
stream.reflector_progress = 0
sent = await stream.upload_to_reflector(host, port)
return sent
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
+1 -1
View File
@@ -48,7 +48,7 @@ async def main(blob_hash: str, url: str):
)
host = host_info[0][4][0]
storage = SQLiteStorage(conf, ":memory:")
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage, conf)
await storage.open()
await blob_manager.setup()
+11 -25
View File
@@ -1,45 +1,31 @@
import sys
import os
import asyncio
import logging
from lbry.blob_exchange.client import request_blob
from lbry.utils import resolve_host
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
from lbry.blob.blob_manager import BlobManager
from lbry.blob_exchange.server import BlobServer
from lbry.schema.address import decode_address
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import Ledger
from lbry.conf import Config
async def main(address: str):
if not Ledger.is_pubkey_address(address):
try:
decode_address(address)
except:
print(f"'{address}' is not a valid lbrycrd address")
return 1
loop = asyncio.get_running_loop()
conf = Config()
async def ensure_blob(blob):
upstream_host, upstream_port = conf.fixed_peers[0]
upstream_host = await resolve_host(upstream_host, upstream_port, 'tcp')
success, proto = await request_blob(loop, blob, upstream_host, int(upstream_port), conf.peer_connect_timeout,
conf.blob_download_timeout)
print(success, proto)
if proto:
proto.close()
storage = SQLiteStorage(conf, os.path.expanduser("/tmp/lbrynet.sqlite"))
storage = SQLiteStorage(os.path.expanduser("~/.lbrynet/lbrynet.sqlite"))
await storage.open()
blob_manager = BlobManager(loop, os.path.expanduser("/tmp/blobfiles"), storage, conf)
blob_manager = BlobManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
await blob_manager.setup()
server = BlobServer(loop, blob_manager, address, blob_callback=ensure_blob)
server = await loop.create_server(
lambda: BlobServer(loop, blob_manager, address),
'0.0.0.0', 4444)
try:
server.start_server(6666, '0.0.0.0')
while True:
await asyncio.sleep(1)
async with server:
await server.serve_forever()
finally:
await storage.close()
@@ -69,6 +69,16 @@ class FileCommands(CommandTestCase):
t = await self.stream_create(f'Stream_{i}', '0.00001')
self.stream_claim_ids.append(t['outputs'][0]['claim_id'])
async def test_file_reflect(self):
tx = await self.stream_create('mirror', '0.01')
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
self.assertEqual([], await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
all_except_sd = [
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
]
await self.reflector.blob_manager.delete_blobs(all_except_sd)
self.assertEqual(all_except_sd, await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
async def test_file_management(self):
await self.stream_create('foo', '0.01')
await self.stream_create('foo2', '0.01')