Compare commits

..
Author SHA1 Message Date
Victor Shyba d14eb24141 always add itself
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (blockchain) (push) Waiting to run
ci / tests / integration (blockchain_legacy_search) (push) Waiting to run
ci / tests / integration (datanetwork) (push) Waiting to run
ci / tests / integration (other) (push) Waiting to run
2021-09-29 23:09:13 -03:00
Victor Shyba 820015872b fix standalone blob server and make it fetch from upstream 2021-09-29 23:04:58 -03:00
Victor Shyba f57b1b3c3d dht: use bytes hex/fromhex instead of binascii 2021-09-29 21:27:53 -03:00
Victor Shyba a85ceb3cca add grin to dht known list 2021-09-29 11:49:53 -03:00
Victor Shyba 254ae66ed3 add madiator to known dht nodes 2021-09-29 00:24:51 -03:00
Victor Shyba 3206fe873e add option to set bootstrap_node 2021-09-28 18:52:23 -03:00
Victor Shyba 63046fce58 add dockerfile for dht node 2021-09-28 18:43:35 -03:00
Victor Shyba b76863d055 configure where to save peers 2021-09-28 18:43:35 -03:00
Victor Shyba d7ed74ab94 define arg types 2021-09-28 18:43:35 -03:00
Victor Shyba 7388f5f1cd add dht seed node script 2021-09-28 18:43:35 -03:00
14 changed files with 54 additions and 103 deletions

No files matched your search

+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.105.0"
__version__ = "0.104.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+3 -8
View File
@@ -6,14 +6,13 @@ log = logging.getLogger(__name__)
class DiskSpaceManager:
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None):
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60):
self.config = config
self.db = db
self.blob_manager = blob_manager
self.cleaning_interval = cleaning_interval
self.running = False
self.task = None
self.analytics = analytics
async def get_space_used_bytes(self):
return await self.db.get_stored_blob_disk_usage()
@@ -22,14 +21,10 @@ class DiskSpaceManager:
return int(await self.get_space_used_bytes()/1024.0/1024.0)
async def clean(self):
space_used_bytes = await self.get_space_used_bytes()
storage_limit = self.config.blob_storage_limit*1024*1024 if self.config.blob_storage_limit else None
if self.analytics:
asyncio.create_task(self.analytics.send_disk_space_used(space_used_bytes, storage_limit))
if not storage_limit:
if not self.config.blob_storage_limit:
return 0
delete = []
available = storage_limit - space_used_bytes
available = (self.config.blob_storage_limit*1024*1024) - await self.get_space_used_bytes()
if available > 0:
return 0
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False):
+9 -3
View File
@@ -18,7 +18,7 @@ MAX_REQUEST_SIZE = 1200
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
idle_timeout: float = 30.0, transfer_timeout: float = 60.0, blob_callback=None):
self.loop = loop
self.blob_manager = blob_manager
self.idle_timeout = idle_timeout
@@ -32,6 +32,7 @@ class BlobServerProtocol(asyncio.Protocol):
self.started_transfer = asyncio.Event(loop=self.loop)
self.transfer_finished = asyncio.Event(loop=self.loop)
self.close_on_idle_task: typing.Optional[asyncio.Task] = None
self.blob_handling_callback: typing.Optional[typing.Callable] = blob_callback
async def close_on_idle(self):
while self.transport:
@@ -92,6 +93,9 @@ class BlobServerProtocol(asyncio.Protocol):
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if self.blob_handling_callback:
await self.blob_handling_callback(blob)
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
@@ -152,7 +156,7 @@ class BlobServerProtocol(asyncio.Protocol):
class BlobServer:
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
idle_timeout: float = 30.0, transfer_timeout: float = 60.0, blob_callback=None):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
@@ -161,6 +165,7 @@ class BlobServer:
self.idle_timeout = idle_timeout
self.transfer_timeout = transfer_timeout
self.server_protocol_class = BlobServerProtocol
self.blob_handling_callback: typing.Optional[typing.Callable] = blob_callback
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -169,7 +174,8 @@ class BlobServer:
async def _start_server():
server = await self.loop.create_server(
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
self.idle_timeout, self.transfer_timeout),
self.idle_timeout, self.transfer_timeout,
blob_callback=self.blob_handling_callback),
interface, port
)
self.started_listening.set()
+1 -2
View File
@@ -96,8 +96,7 @@ class KademliaRPC:
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
]
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address())
peers.append(self.compact_address())
if not peers:
response[PAGE_KEY] = 0
else:
-5
View File
@@ -35,9 +35,6 @@ Code | Name | Message
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
114 | InputStringIsBlank | {argument} cannot be blank.
115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
116 | MissingPublishedFile | File does not exist: {file_path}
**2xx** | Configuration | Configuration errors.
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
@@ -55,7 +52,6 @@ Code | Name | Message
405 | ChannelKeyNotFound | Channel signing key not found.
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
407 | DataDownload | Failed to download blob. *generic*
408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
@@ -63,7 +59,6 @@ Code | Name | Message
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
-39
View File
@@ -84,27 +84,6 @@ class ConflictingInputValueError(InputValueError):
super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
class InputStringIsBlankError(InputValueError):
def __init__(self, argument):
self.argument = argument
super().__init__(f"{argument} cannot be blank.")
class EmptyPublishedFileError(InputValueError):
def __init__(self, file_path):
self.file_path = file_path
super().__init__(f"Cannot publish empty file: {file_path}")
class MissingPublishedFileError(InputValueError):
def __init__(self, file_path):
self.file_path = file_path
super().__init__(f"File does not exist: {file_path}")
class ConfigurationError(BaseError):
"""
Configuration errors.
@@ -228,14 +207,6 @@ class DataDownloadError(WalletError):
super().__init__("Failed to download blob. *generic*")
class PrivateKeyNotFoundError(WalletError):
def __init__(self, key, value):
self.key = key
self.value = value
super().__init__(f"Couldn't find private key for {key} '{value}'.")
class ResolveError(WalletError):
def __init__(self, url):
@@ -287,16 +258,6 @@ class TooManyClaimSearchParametersError(WalletError):
super().__init__(f"{key} cant have more than {limit} items.")
class AlreadyPurchasedError(WalletError):
"""
allow-duplicate-purchase flag to override.
"""
def __init__(self, claim_id_hex):
self.claim_id_hex = claim_id_hex
super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
-9
View File
@@ -18,7 +18,6 @@ DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
DISK_SPACE = 'Disk Space'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
@@ -170,14 +169,6 @@ class AnalyticsManager:
})
)
async def send_disk_space_used(self, storage_used, storage_limit):
await self.track(
self._event(DISK_SPACE, {
'used': storage_used,
'limit': storage_limit,
})
)
async def send_server_startup(self):
await self.track(self._event(SERVER_STARTUP))
+1 -4
View File
@@ -400,10 +400,7 @@ class DiskSpaceComponent(Component):
async def start(self):
db = self.component_manager.get_component(DATABASE_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
self.disk_space_manager = DiskSpaceManager(
self.conf, db, blob_manager,
analytics=self.component_manager.analytics_manager
)
self.disk_space_manager = DiskSpaceManager(self.conf, db, blob_manager)
await self.disk_space_manager.start()
async def stop(self):
+11 -6
View File
@@ -38,7 +38,7 @@ from lbry.dht.peer import make_kademlia_peer
from lbry.error import (
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError,
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError
ConflictingInputValueError
)
from lbry.extras import system_info
from lbry.extras.daemon import analytics
@@ -2295,7 +2295,11 @@ class Daemon(metaclass=JSONRPCServerType):
# TODO: use error from lbry.error
raise Exception("Missing argument claim_id or url.")
if not allow_duplicate_purchase and txo.purchase_receipt:
raise AlreadyPurchasedError(claim_id)
# TODO: use error from lbry.error
raise Exception(
f"You already have a purchase for claim_id '{claim_id}'. "
f"Use --allow-duplicate-purchase flag to override."
)
claim = txo.claim
if not claim.is_stream or not claim.stream.has_fee:
# TODO: use error from lbry.error
@@ -5068,8 +5072,8 @@ class Daemon(metaclass=JSONRPCServerType):
else:
server, port = random.choice(self.conf.reflector_servers)
reflected = await asyncio.gather(*[
self.file_manager.source_managers['stream'].reflect_stream(stream, server, port)
for stream in self.file_manager.get_filtered(**kwargs)
self.file_manager['stream'].reflect_stream(stream, server, port)
for stream in self.file_manager.get_filtered_streams(**kwargs)
])
total = []
for reflected_for_stream in reflected:
@@ -5242,7 +5246,8 @@ class Daemon(metaclass=JSONRPCServerType):
def valid_stream_name_or_error(name: str):
try:
if not name:
raise InputStringIsBlankError('Stream name')
# TODO: use error from lbry.error
raise Exception('Stream name cannot be blank.')
parsed = URL.parse(name)
if parsed.has_channel:
# TODO: use error from lbry.error
@@ -5332,7 +5337,7 @@ class Daemon(metaclass=JSONRPCServerType):
if len(channels) == 1:
if for_signing and not channels[0].has_private_key:
# TODO: use error from lbry.error
raise PrivateKeyNotFoundError(key, value)
raise Exception(f"Couldn't find private key for {key} '{value}'. ")
return channels[0]
elif len(channels) > 1:
# TODO: use error from lbry.error
+2 -3
View File
@@ -10,7 +10,6 @@ from google.protobuf.json_format import MessageToDict
from lbry.crypto.base58 import Base58
from lbry.constants import COIN
from lbry.error import MissingPublishedFileError, EmptyPublishedFileError
from lbry.schema.mime_types import guess_media_type
from lbry.schema.base import Metadata, BaseMessageList
@@ -140,10 +139,10 @@ class Source(Metadata):
self.name = os.path.basename(file_path)
self.media_type, stream_type = guess_media_type(file_path)
if not os.path.isfile(file_path):
raise MissingPublishedFileError(file_path)
raise Exception(f"File does not exist: {file_path}")
self.size = os.path.getsize(file_path)
if self.size == 0:
raise EmptyPublishedFileError(file_path)
raise Exception(f"Cannot publish empty file: {file_path}")
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
return stream_type
-1
View File
@@ -228,7 +228,6 @@ class StreamManager(SourceManager):
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
stream.reflector_progress = 0
sent = await stream.upload_to_reflector(host, port)
return sent
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
+1 -1
View File
@@ -48,7 +48,7 @@ async def main(blob_hash: str, url: str):
)
host = host_info[0][4][0]
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
storage = SQLiteStorage(conf, ":memory:")
blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage, conf)
await storage.open()
await blob_manager.setup()
+25 -11
View File
@@ -1,31 +1,45 @@
import sys
import os
import asyncio
import logging
from lbry.blob_exchange.client import request_blob
from lbry.utils import resolve_host
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
from lbry.blob.blob_manager import BlobManager
from lbry.blob_exchange.server import BlobServer
from lbry.schema.address import decode_address
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import Ledger
from lbry.conf import Config
async def main(address: str):
try:
decode_address(address)
except:
if not Ledger.is_pubkey_address(address):
print(f"'{address}' is not a valid lbrycrd address")
return 1
loop = asyncio.get_running_loop()
conf = Config()
storage = SQLiteStorage(os.path.expanduser("~/.lbrynet/lbrynet.sqlite"))
async def ensure_blob(blob):
upstream_host, upstream_port = conf.fixed_peers[0]
upstream_host = await resolve_host(upstream_host, upstream_port, 'tcp')
success, proto = await request_blob(loop, blob, upstream_host, int(upstream_port), conf.peer_connect_timeout,
conf.blob_download_timeout)
print(success, proto)
if proto:
proto.close()
storage = SQLiteStorage(conf, os.path.expanduser("/tmp/lbrynet.sqlite"))
await storage.open()
blob_manager = BlobManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
blob_manager = BlobManager(loop, os.path.expanduser("/tmp/blobfiles"), storage, conf)
await blob_manager.setup()
server = await loop.create_server(
lambda: BlobServer(loop, blob_manager, address),
'0.0.0.0', 4444)
server = BlobServer(loop, blob_manager, address, blob_callback=ensure_blob)
try:
async with server:
await server.serve_forever()
server.start_server(6666, '0.0.0.0')
while True:
await asyncio.sleep(1)
finally:
await storage.close()
@@ -69,16 +69,6 @@ class FileCommands(CommandTestCase):
t = await self.stream_create(f'Stream_{i}', '0.00001')
self.stream_claim_ids.append(t['outputs'][0]['claim_id'])
async def test_file_reflect(self):
tx = await self.stream_create('mirror', '0.01')
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
self.assertEqual([], await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
all_except_sd = [
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
]
await self.reflector.blob_manager.delete_blobs(all_except_sd)
self.assertEqual(all_except_sd, await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
async def test_file_management(self):
await self.stream_create('foo', '0.01')
await self.stream_create('foo2', '0.01')