Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69e7566eaa | ||
|
|
399ac561c4 | ||
|
|
588568752b | ||
|
|
a30e2daf05 | ||
|
|
adb7edd3b2 | ||
|
|
06a9770504 | ||
|
|
0da43d3e05 | ||
|
|
077dbb3138 | ||
|
|
3b8fd65fa9 | ||
|
|
d0ddaa5518 | ||
|
|
a60ab97552 | ||
|
|
078d09976c | ||
|
|
3770558694 | ||
|
|
4409b8c4f8 | ||
|
|
f5b7aaac18 | ||
|
|
6dbb6b9079 | ||
|
|
16b774a69e | ||
|
|
72ea19532e | ||
|
|
ad24946af3 | ||
|
|
b676907fd7 | ||
|
|
6965971613 | ||
|
|
6651111522 | ||
|
|
d8630a02d9 | ||
|
|
69345b4b0e | ||
|
|
f23aea9951 | ||
|
|
f96c46e84a | ||
|
|
bae2939943 | ||
|
|
7063138e3c | ||
|
|
798ba9acf8 | ||
|
|
c8dd1987e6 | ||
|
|
8d2c9e5785 | ||
|
|
9dbf47916b | ||
|
|
565ef5e454 | ||
|
|
9f1f2b1781 | ||
|
|
76df239d1a | ||
|
|
86ee6a78f5 | ||
|
|
4fb6f2bb27 | ||
|
|
ec039825e8 | ||
|
|
a10f09d169 | ||
|
|
76409600e1 | ||
|
|
d06a0f876a | ||
|
|
8e857de62a | ||
|
|
3d37f7c735 | ||
|
|
552b94bc65 | ||
|
|
17164601a1 | ||
|
|
26dd6d2ac7 | ||
|
|
5d5e36a2c2 | ||
|
|
77afbcded6 | ||
|
|
ae11d72f66 |
No files matched your search
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
|
||||
- App: https://github.com/lbryio/lbry-desktop/releases
|
||||
- Daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Discord at https://chat.lbry.io
|
||||
We are also available on Discord at https://chat.lbry.com
|
||||
-->
|
||||
|
||||
|
||||
|
||||
+6
-4
@@ -4,7 +4,9 @@ language: python
|
||||
python: "3.7"
|
||||
|
||||
env:
|
||||
- TORBA=master
|
||||
global:
|
||||
# must also be updated in wine_build.sh
|
||||
- TORBA=master
|
||||
|
||||
jobs:
|
||||
include:
|
||||
@@ -59,7 +61,7 @@ jobs:
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh ${TORBA}
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
@@ -125,7 +127,7 @@ jobs:
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
@@ -133,7 +135,7 @@ jobs:
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
|
||||
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.38.0rc4"
|
||||
__version__ = "0.38.0rc6"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -5,6 +5,7 @@ import logging
|
||||
from lbrynet.utils import LRUCache
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
@@ -33,6 +34,7 @@ class BlobManager:
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
@@ -84,9 +86,11 @@ class BlobManager:
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
self.connection_manager.start()
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
self.connection_manager.stop()
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
|
||||
@@ -8,18 +8,20 @@ from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
|
||||
self.peer_timeout = peer_timeout
|
||||
self.connection_manager = connection_manager
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
@@ -31,6 +33,12 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.connection_manager:
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
# assert self.peer_address is not None
|
||||
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
|
||||
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
@@ -94,10 +102,15 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
if self.connection_manager:
|
||||
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
@@ -186,11 +199,16 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
addr = transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr[0], addr[1]
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
if self.connection_manager:
|
||||
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
if self.connection_manager:
|
||||
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
@@ -199,16 +217,19 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
protocol = BlobExchangeClientProtocol(
|
||||
loop, blob_download_timeout, connection_manager
|
||||
)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
protocol.transport = connected_transport
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
|
||||
@@ -41,7 +41,9 @@ class BlobDownloader:
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
|
||||
connection_manager=self.blob_manager.connection_manager
|
||||
|
||||
)
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
|
||||
@@ -22,15 +22,23 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.peer_address_and_port: typing.Optional[str] = None
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
|
||||
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
|
||||
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
self.transport.write(BlobResponse(to_send).serialize())
|
||||
serialized = BlobResponse(to_send).serialize()
|
||||
self.transport.write(serialized)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
@@ -72,6 +80,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
@@ -97,7 +106,7 @@ class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
+20
-15
@@ -121,7 +121,7 @@ class Toggle(Setting[bool]):
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Opposite of --{self.cli_name}",
|
||||
help=f"Opposite of {self.cli_name}",
|
||||
dest=self.name,
|
||||
action="store_false",
|
||||
default=NOT_SET
|
||||
@@ -192,7 +192,21 @@ class MaxKeyFee(Setting[dict]):
|
||||
)
|
||||
|
||||
|
||||
class Servers(Setting[list]):
|
||||
class ListSetting(Setting[list]):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
action='append'
|
||||
)
|
||||
|
||||
|
||||
class Servers(ListSetting):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
@@ -225,16 +239,8 @@ class Servers(Setting[list]):
|
||||
return [f"{host}:{port}" for host, port in value]
|
||||
return value
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
nargs="*",
|
||||
help=self.doc,
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class Strings(Setting[list]):
|
||||
class Strings(ListSetting):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
@@ -269,7 +275,7 @@ class ArgumentAccess:
|
||||
def load(self, args):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = getattr(args, setting.name, NOT_SET)
|
||||
if value != NOT_SET:
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.args[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
@@ -520,7 +526,7 @@ class Config(CLIConfig):
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.com', 50001), # US EAST
|
||||
@@ -534,8 +540,7 @@ class Config(CLIConfig):
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
# comment server uses a single string because requests are made to the /api resource
|
||||
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import collections
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CONNECTED_EVENT = "connected"
|
||||
DISCONNECTED_EVENT = "disconnected"
|
||||
TRANSFERRED_EVENT = "transferred"
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self.loop = loop
|
||||
self.incoming_connected: typing.Set[str] = set()
|
||||
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self.outgoing_connected: typing.Set[str] = set()
|
||||
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self._status = {}
|
||||
|
||||
self._task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self._status
|
||||
|
||||
def sent_data(self, host_and_port: str, size: int):
|
||||
self.outgoing[host_and_port] += size
|
||||
|
||||
def received_data(self, host_and_port: str, size: int):
|
||||
self.incoming[host_and_port] += size
|
||||
|
||||
def connection_made(self, host_and_port: str):
|
||||
self.outgoing_connected.add(host_and_port)
|
||||
|
||||
def connection_received(self, host_and_port: str):
|
||||
# self.incoming_connected.add(host_and_port)
|
||||
pass
|
||||
|
||||
def outgoing_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.outgoing_connected:
|
||||
self.outgoing_connected.remove(host_and_port)
|
||||
|
||||
def incoming_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.incoming_connected:
|
||||
self.incoming_connected.remove(host_and_port)
|
||||
|
||||
async def _update(self):
|
||||
|
||||
self._status = {
|
||||
'incoming_bps': {},
|
||||
'outgoing_bps': {},
|
||||
'total_incoming_mbs': 0.0,
|
||||
'total_outgoing_mbs': 0.0,
|
||||
'time': self.loop.time()
|
||||
}
|
||||
|
||||
while True:
|
||||
last = self.loop.time()
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
self._status['incoming_bps'].clear()
|
||||
self._status['outgoing_bps'].clear()
|
||||
while self.outgoing:
|
||||
k, v = self.outgoing.popitem()
|
||||
self._status['outgoing_bps'][k] = v
|
||||
while self.incoming:
|
||||
k, v = self.incoming.popitem()
|
||||
self._status['incoming_bps'][k] = v
|
||||
now = self.loop.time()
|
||||
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['time'] = now
|
||||
|
||||
def stop(self):
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
self.outgoing.clear()
|
||||
self.outgoing_connected.clear()
|
||||
self.incoming.clear()
|
||||
self.incoming_connected.clear()
|
||||
self._status.clear()
|
||||
|
||||
def start(self):
|
||||
self.stop()
|
||||
self._task = self.loop.create_task(self._update())
|
||||
@@ -169,7 +169,7 @@ def add_command_parser(parent, command):
|
||||
|
||||
def get_argument_parser():
|
||||
main = ArgumentParser(
|
||||
'lbrynet', description='An interface to the LBRY Network.'
|
||||
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
|
||||
)
|
||||
main.add_argument(
|
||||
'-v', '--version', dest='cli_version', action="store_true",
|
||||
@@ -192,8 +192,8 @@ def get_argument_parser():
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
start.set_defaults(command='start', start_parser=start)
|
||||
Config.contribute_to_argparse(start)
|
||||
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
|
||||
|
||||
api = Daemon.get_api_definitions()
|
||||
groups = {}
|
||||
@@ -239,6 +239,7 @@ def main(argv=None):
|
||||
if args.help:
|
||||
args.start_parser.print_help()
|
||||
return 0
|
||||
|
||||
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
conf.log_file_path, maxBytes=2097152, backupCount=5
|
||||
@@ -264,6 +265,7 @@ def main(argv=None):
|
||||
loop.set_debug(True)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
|
||||
@@ -278,7 +278,7 @@ class BlobComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager: BlobManager = None
|
||||
self.blob_manager: typing.Optional[BlobManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
@@ -294,7 +294,7 @@ class BlobComponent(Component):
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
|
||||
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
@@ -304,7 +304,10 @@ class BlobComponent(Component):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = len(self.blob_manager.completed_blob_hashes)
|
||||
return {'finished_blobs': count}
|
||||
return {
|
||||
'finished_blobs': count,
|
||||
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
|
||||
}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -405,7 +408,7 @@ class StreamManagerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: StreamManager = None
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
@@ -415,7 +418,7 @@ class StreamManagerComponent(Component):
|
||||
if not self.stream_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams)
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -444,7 +447,7 @@ class PeerProtocolServerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_server: BlobServer = None
|
||||
self.blob_server: typing.Optional[BlobServer] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobServer]:
|
||||
@@ -510,7 +513,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
|
||||
+114
-159
@@ -33,13 +33,13 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
|
||||
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
from lbrynet.extras.daemon.comment_client import jsonrpc_post, sign_comment
|
||||
from lbrynet.extras.daemon.undecorated import undecorated
|
||||
from lbrynet.wallet.transaction import Transaction, Output, Input
|
||||
from lbrynet.wallet.account import Account as LBCAccount
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.schema.url import URL
|
||||
from lbrynet.extras.daemon.comment_client import jsonrpc_batch, jsonrpc_post, rpc_body
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -51,7 +51,6 @@ if typing.TYPE_CHECKING:
|
||||
from lbrynet.wallet.manager import LbryWalletManager
|
||||
from lbrynet.wallet.ledger import MainNetLedger
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -746,6 +745,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
},
|
||||
'blob_manager': {
|
||||
'finished_blobs': (int) number of finished blobs in the blob manager,
|
||||
'connections': {
|
||||
'incoming_bps': {
|
||||
<source ip and tcp port>: (int) bytes per second received,
|
||||
},
|
||||
'outgoing_bps': {
|
||||
<destination ip and tcp port>: (int) bytes per second sent,
|
||||
},
|
||||
'total_outgoing_mps': (float) megabytes per second sent,
|
||||
'total_incoming_mps': (float) megabytes per second received,
|
||||
'time': (float) timestamp
|
||||
}
|
||||
},
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
@@ -797,16 +807,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns:
|
||||
(dict) Dictionary of lbry version information
|
||||
{
|
||||
'build': (str) build type (e.g. "dev", "rc", "release"),
|
||||
'ip': (str) remote ip, if available,
|
||||
'lbrynet_version': (str) lbrynet_version,
|
||||
'lbryum_version': (str) lbryum_version,
|
||||
'lbryschema_version': (str) lbryschema_version,
|
||||
'os_release': (str) os release string
|
||||
'os_system': (str) os name
|
||||
'platform': (str) platform string
|
||||
'processor': (str) processor type,
|
||||
'python_version': (str) python version,
|
||||
'platform': (str) platform string,
|
||||
'os_release': (str) os release string,
|
||||
'os_system': (str) os name,
|
||||
'lbrynet_version': (str) lbrynet version,
|
||||
'torba_version': (str) torba version,
|
||||
'build': (str) "dev" | "qa" | "rc" | "release",
|
||||
}
|
||||
"""
|
||||
platform_info = system_info.get_platform()
|
||||
@@ -1697,7 +1705,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
claim_search [<name> | --name=<name>] [--claim_id=<claim_id>] [--txid=<txid>] [--nout=<nout>]
|
||||
[--channel=<channel> | --channel_ids=<channel_ids>...]
|
||||
[--valid_channel_signatures] [--invalid_channel_signatures]
|
||||
[--is_controlling] [--release_time=<release_time>]
|
||||
[--is_controlling] [--release_time=<release_time>] [--public_key_id=<public_key_id>]
|
||||
[--timestamp=<timestamp>] [--creation_timestamp=<creation_timestamp>]
|
||||
[--height=<height>] [--creation_height=<creation_height>]
|
||||
[--activation_height=<activation_height>] [--expiration_height=<expiration_height>]
|
||||
@@ -1705,6 +1713,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--support_amount=<support_amount>] [--trending_group=<trending_group>]
|
||||
[--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>]
|
||||
[--trending_global=<trending_global]
|
||||
[--claim_type=<claim_type>] [--stream_types=<stream_types>...] [--media_types=<media_types>...]
|
||||
[--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
|
||||
[--any_languages=<any_languages>...] [--all_languages=<all_languages>...]
|
||||
[--not_languages=<not_languages>...]
|
||||
@@ -1730,6 +1739,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--valid_channel_signatures : (bool) only return claims with valid channel signatures
|
||||
--invalid_channel_signatures : (bool) only return claims with invalid channel signatures
|
||||
--is_controlling : (bool) only return winning claims of their respective name
|
||||
--public_key_id=<public_key_id> : (str) only return channels having this public key id, this is
|
||||
the same key as used in the wallet file to map
|
||||
channel certificate private keys: {'public_key_id': 'private key'}
|
||||
--height=<height> : (int) last updated block height (supports equality constraints)
|
||||
--timestamp=<timestamp> : (int) last updated timestamp (supports equality constraints)
|
||||
--creation_height=<creation_height> : (int) created at block height (supports equality constraints)
|
||||
@@ -1769,6 +1781,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--trending_global=<trending_global>: (int) trending value calculated relative to all
|
||||
trending content globally (supports
|
||||
equality constraints)
|
||||
--claim_type=<claim_type> : (str) filter by 'channel', 'stream' or 'unknown'
|
||||
--stream_types=<stream_types> : (list) filter by 'video', 'image', 'document', etc
|
||||
--media_types=<media_types> : (list) filter by 'video/mp4', 'image/png', etc
|
||||
--any_tags=<any_tags> : (list) find claims containing any of the tags
|
||||
--all_tags=<all_tags> : (list) find claims containing every tag
|
||||
--not_tags=<not_tags> : (list) find claims not containing any of these tags
|
||||
@@ -1790,9 +1805,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
if kwargs.pop('valid_channel_signatures', False):
|
||||
kwargs['is_channel_signature_valid'] = 1
|
||||
elif kwargs.pop('invalid_channel_signatures', False):
|
||||
kwargs['is_channel_signature_valid'] = 0
|
||||
kwargs['signature_valid'] = 1
|
||||
if kwargs.pop('invalid_channel_signatures', False):
|
||||
kwargs['signature_valid'] = 0
|
||||
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', 10)), 50)
|
||||
kwargs.update({'offset': page_size * (page_num-1), 'limit': page_size})
|
||||
txos, offset, total = await self.ledger.claim_search(**kwargs)
|
||||
@@ -2278,7 +2293,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--duration=<duration> : (int) audio/video duration in seconds, automatically calculated
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel
|
||||
--channel_name=<channel_name> : (str) name of publisher channel
|
||||
--channel_account_id=<channel_id>: (str) one or more account ids for accounts to look in
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
--account_id=<account_id> : (str) account to use for funding the transaction
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
@@ -2391,7 +2406,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--duration=<duration> : (int) audio/video duration in seconds, automatically calculated
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel
|
||||
--channel_account_id=<channel_id>: (str) one or more account ids for accounts to look in
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
--account_id=<account_id> : (str) account to use for funding the transaction
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
@@ -2541,7 +2556,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel
|
||||
--clear_channel : (bool) remove channel signature
|
||||
--channel_account_id=<channel_id>: (str) one or more account ids for accounts to look in
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
--account_id=<account_id> : (str) account to use for funding the transaction
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
@@ -3328,173 +3343,113 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Create and list comments.
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_comment_list(self, claim_id, parent_comment_id=None, flat=False,
|
||||
page=1, page_size=None, max_replies_shown=5):
|
||||
async def jsonrpc_comment_list(self, claim_id: str, parent_id: str = None,
|
||||
page=1, page_size=50, include_replies=True):
|
||||
"""
|
||||
List comments associated with a claim.
|
||||
|
||||
Usage:
|
||||
comment_list <claim_id> [--flat] [(--page=<page> --page_size=<page_size>)]
|
||||
[--parent_comment_id=<parent_comment_id>]
|
||||
[--max_replies_shown=<max_replies_shown>]
|
||||
comment_list (<claim_id> | --claim_id=<claim_id>)
|
||||
[(--page=<page> --page_size=<page_size>)]
|
||||
[--parent_id=<parent_id>] [--include_replies]
|
||||
|
||||
Options:
|
||||
--flat : (bool) Flag to indicate whether or not you want the
|
||||
replies to be flattened along with the rest of
|
||||
the comments attached to the claim. Off by default
|
||||
--parent_comment_id=<parent_comment_id> : (int) The ID of an existing
|
||||
comment to list replies from
|
||||
--max_replies_shown=<max_replies_shown> : (int) For every comment that we pull replies from,
|
||||
only retrieve up to this amount.
|
||||
Note: This is not the same as page size.
|
||||
--claim_id=<claim_id> : (str) The claim on which the comment will be made on
|
||||
--include_replies : (bool) Flag to indicate whether or not you want the
|
||||
replies to be included with the response or not
|
||||
--parent_id=<parent_id> : (int)
|
||||
|
||||
--page=<page> : (int) The page you'd like to see in the comment list.
|
||||
The first page is 1, second page is 2, and so on.
|
||||
--page_size=<page_size> : (int) The amount of comments that you'd like to
|
||||
retrieve in one request
|
||||
|
||||
Returns:
|
||||
(dict) Dict containing the following schema:
|
||||
(dict) Containing the list, and information about the paginated content:
|
||||
{
|
||||
"page": (int) The page of comments as provided when limiting response to page_size.
|
||||
"page_size": (int) Number of comments in the given page. -1 if page_size wasn't used
|
||||
"comments": (list) Contains all the comments (as dicts) as provided by the specified parameters
|
||||
"page": "Page number of the current items.",
|
||||
"page_size": "Number of items to show on a page.",
|
||||
"total_pages": "Total number of pages.",
|
||||
"total_items": "Total number of items.",
|
||||
"items": "A List of dict objects representing comments."
|
||||
[
|
||||
{
|
||||
"comment": (str) The actual string as inputted by the user,
|
||||
"comment_id": (str) The Comment's unique identifier,
|
||||
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
|
||||
"channel_id": (str) The Channel Claim ID that this comment was posted under,
|
||||
"signature": (str) The signature of the comment,
|
||||
"channel_url": (str) Channel's URI in the ClaimTrie,
|
||||
"parent_id": (str) Comment this is replying to, (None) if this is the root,
|
||||
"timestamp": (int) The time at which comment was entered into the server at, in nanoseconds.
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
# Should be like this:
|
||||
# comment list [claim_id] [parent_comment_id] --flat --page=1 --page-size=10
|
||||
url = self.conf.comment_server
|
||||
# The server uses permanent URIs for keys; not claims.
|
||||
# This is temporary until we can get that functionality removed
|
||||
claim_info = (await self.jsonrpc_claim_search(claim_id=claim_id))
|
||||
if 'error' in claim_info:
|
||||
raise Exception(claim_info['error'])
|
||||
if claim_info["page"] == 0:
|
||||
return {'page': 1, 'page_size': 0, 'comments': []}
|
||||
claim_uri = claim_info["items"][0].permanent_url
|
||||
# These two cases need separation since getting replies requires a bit of magic
|
||||
# to reduce request count from O(n^2) to O(1)
|
||||
if parent_comment_id:
|
||||
# Since we don't directly get all the comment data at once,
|
||||
# we have to do a bit more work to get them
|
||||
comment_ids = await jsonrpc_post(url, 'get_comment_replies',
|
||||
comm_index=parent_comment_id, clean=False)
|
||||
comment_ids = comment_ids['result']
|
||||
if page_size is not None:
|
||||
comment_ids = comment_ids[page_size * (page - 1): page_size * page]
|
||||
# now we have to just batch request the reply comments
|
||||
comments_batch = [
|
||||
rpc_body('get_comment_data', index, comm_index=comment_id, better_keys=True)
|
||||
for index, comment_id in enumerate(comment_ids)
|
||||
]
|
||||
del comment_ids
|
||||
comments = await jsonrpc_batch(url, comments_batch, clean=True)
|
||||
else:
|
||||
# Get the content of the top level comments
|
||||
comments = await jsonrpc_post(url, 'get_claim_comments', uri=claim_uri, better_keys=True)
|
||||
if page_size is not None:
|
||||
comments = comments[page_size * (page - 1): page_size * page]
|
||||
# By now comments should be a list containing comment dicts that are supposed to be
|
||||
# at the given height that was requested. The parent_id may or may not be present
|
||||
# in the dicts, as they may or may not be replies to comments at a higher level
|
||||
# However this is dependent purely on whether or not parent_comment_id is None or not
|
||||
reply_lists = await jsonrpc_batch(url, [
|
||||
rpc_body('get_comment_replies', index, comm_index=comment['comment_id'])
|
||||
for index, comment in enumerate(comments)
|
||||
])
|
||||
response = {
|
||||
'page': page,
|
||||
'page_size': -1 if page_size is None else page_size,
|
||||
'comments': []
|
||||
}
|
||||
if flat:
|
||||
# If it's flat then we'll need to get the comments into an order such that
|
||||
# If an element e in the list has a non-null parent id, the element before it
|
||||
# is either also a reply with the same parent id, or has an id that equals e's parent id,
|
||||
# in which case it's the comment that is being replied to.
|
||||
# Otherwise, if it has a null parent id, then it is a top level comment.
|
||||
return await jsonrpc_post(
|
||||
self.conf.comment_server,
|
||||
"get_claim_comments",
|
||||
claim_id=claim_id,
|
||||
parent_id=parent_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
top_level=not include_replies
|
||||
)
|
||||
|
||||
# To do this, we create a dict that maps the index of the comment in the array
|
||||
# to a list containing the comment IDs of the replies
|
||||
comment_replies = {resp['id']: resp['result'] for resp in reply_lists if 'result' in resp}
|
||||
|
||||
# Next, we create a batch request for the actual data of all of the replies
|
||||
# the id in this batch request is going to be in the form 'X:Y'
|
||||
# where X is the index of the parent comment in `comments,
|
||||
# and Y is index of the reply's ID within the list X maps to in `comment_replies`
|
||||
full_replies_batch = [
|
||||
rpc_body('get_comment_data', f'{parent_idx}:{idx}', comm_index=reply_id, better_keys=True)
|
||||
for parent_idx, id_list in comment_replies.items()
|
||||
for idx, reply_id in enumerate(id_list[0:max_replies_shown])
|
||||
]
|
||||
reply_dump = await jsonrpc_batch(url, full_replies_batch)
|
||||
del full_replies_batch
|
||||
# This neatly orders the response into a dict to aggregate the
|
||||
# full comments by the parent comment they're replying to
|
||||
#
|
||||
# WARNING: The following block is going to be saving the comment dict
|
||||
# objects TO `comment_replies`. This means that the lists
|
||||
# stored in `comments_replies` may not hold just comments, but
|
||||
# the ids of the comments who weren't requested due to the
|
||||
# maximum reply limit. They need to be either cleaned out or stored
|
||||
# somewhere else
|
||||
|
||||
for comment in reply_dump:
|
||||
parent_index, reply_index = comment['id'].split(':')
|
||||
parent_index, reply_index = int(parent_index), int(reply_index)
|
||||
comment_replies[parent_index][reply_index] = comment['result']
|
||||
|
||||
for idx, parent_comment in enumerate(comments):
|
||||
if 'parent_id' not in parent_comment:
|
||||
parent_comment['parent_id'] = None
|
||||
parent_comment['reply_count'] = len(comment_replies[idx])
|
||||
parent_comment['omitted'] = 0
|
||||
if len(comment_replies[idx]) > max_replies_shown:
|
||||
parent_comment['omitted'] = len(comment_replies[idx]) - max_replies_shown
|
||||
|
||||
response['comments'].append(parent_comment)
|
||||
response['comments'] += comment_replies[idx][0:max_replies_shown]
|
||||
response['page_size'] = page_size if page_size is not None else -1
|
||||
return response
|
||||
else:
|
||||
for id_list in reply_lists:
|
||||
comments[id_list['id']]['reply_count'] = len(id_list['result'])
|
||||
comments[id_list['id']]['omitted'] = len(id_list['result'])
|
||||
response['comments'] = comments
|
||||
del reply_lists
|
||||
return response
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_comment_create(self, claim_id: str, channel_id: str,
|
||||
message: str, parent_comment_id: int = None) -> dict:
|
||||
async def jsonrpc_comment_create(
|
||||
self, claim_id: str, comment: str, parent_id: str = None,
|
||||
channel_account_id=None, channel_name: str = None, channel_id: str = None) -> dict:
|
||||
"""
|
||||
Create and associate a comment with a claim using your channel identity.
|
||||
|
||||
Usage:
|
||||
comment_create <claim_id> <channel_id> <message> [--parent_comment_id=<parent_comment_id>]
|
||||
comment_create (<comment> | --comment=<comment>)
|
||||
(<claim_id> | --claim_id=<claim_id>)
|
||||
[--parent_id=<parent_id>]
|
||||
[--channel_id=<channel_id>] [--channel_name=<channel_name>]
|
||||
[--channel_account_id=<channel_account_id>...]
|
||||
|
||||
Options:
|
||||
--parent_comment_id=<parent_comment_id> : (int) The ID of a comment to make a response to
|
||||
--comment=<comment> : (str) Comment to be made, should be at most 2000 characters.
|
||||
--claim_id=<claim_id> : (str) The ID of the claim on which the comment should be made on
|
||||
--parent_id=<parent_id> : (str) The ID of a comment to make a response to
|
||||
--channel_id=<channel_id> : (str) The ID of the channel you want to post under
|
||||
--channel_name=<channel_name>: (str) The channel you want to post as, prepend with a '@'
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
|
||||
Returns:
|
||||
(dict) Comment object if successfully made
|
||||
(dict) Comment object if successfully made, (None) otherwise
|
||||
{
|
||||
"comment": (str) The actual string as inputted by the user,
|
||||
"comment_id": (str) The Comment's unique identifier,
|
||||
"channel_name": (str) Name of the channel this was posted under, prepended with a '@',
|
||||
"channel_id": (str) The Channel Claim ID that this comeent was posted under,
|
||||
"signature": (str) The signature of the comment,
|
||||
"channel_url": (str) Channel's URI in the ClaimTrie,
|
||||
"parent_id": (str) Comment this is replying to, (None) if this is the root,
|
||||
"timestamp": (int) The time at which comment was entered into the server at, in nanoseconds.
|
||||
}
|
||||
"""
|
||||
if not 1 < len(message) <= 2000:
|
||||
raise Exception(f'Message length ({len(message)}) needs to be between 2 and 2000 chars')
|
||||
url = self.conf.comment_server
|
||||
if parent_comment_id is not None:
|
||||
comment_id = await jsonrpc_post(url, 'reply', parent_id=parent_comment_id,
|
||||
poster=channel_id, message=message)
|
||||
else:
|
||||
claim_data = await self.jsonrpc_claim_search(claim_id=claim_id)
|
||||
if 'error' not in claim_data and claim_data['total_pages'] == 1:
|
||||
uri = claim_data['items'][0].permanent_url
|
||||
comment_id = await jsonrpc_post(url, 'comment', uri=uri,
|
||||
poster=channel_id, message=message)
|
||||
else:
|
||||
raise Exception(f"permanent_url is not in the claim_data {claim_data}\n"
|
||||
f"The given claim_id ({claim_id}) may be invalid")
|
||||
return await jsonrpc_post(url, 'get_comment_data', comm_index=comment_id, better_keys=True)
|
||||
comment = {
|
||||
'comment': comment,
|
||||
'claim_id': claim_id,
|
||||
'parent_id': parent_id,
|
||||
}
|
||||
channel = await self.get_channel_or_none(channel_account_id, channel_id, channel_name, for_signing=True)
|
||||
if channel:
|
||||
comment.update({
|
||||
'channel_id': channel.claim_id,
|
||||
'channel_name': channel.claim_name,
|
||||
'signature': sign_comment(
|
||||
channel_name=channel_name,
|
||||
channel_id=channel_id,
|
||||
comment=comment,
|
||||
salt=time.time_ns()
|
||||
)
|
||||
})
|
||||
return await jsonrpc_post(self.conf.comment_server, 'create_comment', **comment)
|
||||
|
||||
def valid_address_or_error(self, address):
|
||||
try:
|
||||
|
||||
@@ -1,31 +1,51 @@
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
import hashlib
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import padding
|
||||
from cryptography.hazmat.primitives.asymmetric import utils
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sign_comment(**kwargs):
|
||||
private_key = generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=4096,
|
||||
backend=default_backend()
|
||||
)
|
||||
chosen_hash = hashes.SHA256()
|
||||
hasher = hashes.Hash(chosen_hash, default_backend())
|
||||
value_to_hash = b':'.join(bytes(v, 'utf-8') for v in kwargs.values() if type(v) is str)
|
||||
hasher.update(value_to_hash)
|
||||
digest = hasher.finalize()
|
||||
signature = private_key.sign(
|
||||
digest,
|
||||
padding.PSS(
|
||||
mgf=padding.MGF1(hashes.SHA256()),
|
||||
salt_length=padding.PSS.MAX_LENGTH
|
||||
),
|
||||
utils.Prehashed(chosen_hash)
|
||||
)
|
||||
m = hashlib.sha3_256()
|
||||
m.update(signature)
|
||||
return m.hexdigest()
|
||||
|
||||
|
||||
def rpc_body(method: str, rpc_id: any, **params) -> dict:
|
||||
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
clean = params.pop('clean', True)
|
||||
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
|
||||
if clean:
|
||||
if 'error' in response:
|
||||
return response['error']
|
||||
return response['result']
|
||||
else:
|
||||
return response
|
||||
|
||||
|
||||
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
complete = []
|
||||
batch_size = max(batch_size, 50)
|
||||
for i in range(0, len(calls), batch_size):
|
||||
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
|
||||
complete += await response.json()
|
||||
if clean:
|
||||
complete = [body['result'] if 'result' in body else None for body in complete]
|
||||
return complete
|
||||
async with aiohttp.request('POST', url, json=json_body, headers=headers) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
except aiohttp.client.ContentTypeError as cte:
|
||||
log.exception('Unable to decode respose from server: %s', cte)
|
||||
return await response.text()
|
||||
@@ -123,8 +123,8 @@ class LBRYioFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"BTCLBC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
"lbry.com",
|
||||
"https://api.lbry.com/lbc/exchange_rate",
|
||||
{},
|
||||
0.0,
|
||||
)
|
||||
@@ -140,8 +140,8 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"USDBTC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
"lbry.com",
|
||||
"https://api.lbry.com/lbc/exchange_rate",
|
||||
{},
|
||||
0.0,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from binascii import hexlify
|
||||
from binascii import hexlify, unhexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
|
||||
@@ -93,10 +94,16 @@ def encode_file_doc():
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'claim': '(Claim) None if claim is not found',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'protobuf': '(str) hex encoded Stream protobuf',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'content_fee': '(Transaction) None if content claim did not have a fee',
|
||||
'height': '(int) height of the claim',
|
||||
'confirmations': '(int) number of blocks since the claim was confirmed',
|
||||
'timestamp': '(int) timestamp of the block containing the claim'
|
||||
}
|
||||
|
||||
|
||||
@@ -184,12 +191,13 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
if txo.channel:
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
except DecodeError:
|
||||
pass
|
||||
return output
|
||||
@@ -217,16 +225,74 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return result
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
file = managed_stream.as_dict()
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
file.update({
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
})
|
||||
return file
|
||||
full_path = managed_stream.full_path
|
||||
file_name = managed_stream.file_name
|
||||
download_directory = managed_stream.download_directory
|
||||
if managed_stream.full_path and managed_stream.output_file_exists:
|
||||
if managed_stream.written_bytes:
|
||||
written_bytes = managed_stream.written_bytes
|
||||
else:
|
||||
written_bytes = os.stat(managed_stream.full_path).st_size
|
||||
else:
|
||||
full_path = None
|
||||
file_name = None
|
||||
download_directory = None
|
||||
written_bytes = None
|
||||
|
||||
@staticmethod
|
||||
def encode_claim(claim):
|
||||
return getattr(claim, claim.claim_type).to_dict()
|
||||
if managed_stream.stream_claim_info:
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
timestamp = self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
confirmations = None if not tx_height else ((best_height + 1) - tx_height if tx_height > 0 else tx_height)
|
||||
else:
|
||||
tx_height = None
|
||||
timestamp = None
|
||||
confirmations = None
|
||||
return {
|
||||
'streaming_url': f"http://{managed_stream.config.streaming_host}:{managed_stream.config.streaming_port}"
|
||||
f"/stream/{managed_stream.sd_hash}",
|
||||
'completed': (managed_stream.output_file_exists and (managed_stream.status in ('stopped', 'finished'))
|
||||
or not managed_stream.saving.is_set()) or all(
|
||||
managed_stream.blob_manager.is_blob_verified(b.blob_hash)
|
||||
for b in managed_stream.descriptor.blobs[:-1]),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'download_path': full_path,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
'nout': managed_stream.nout,
|
||||
'outpoint': managed_stream.outpoint,
|
||||
'claim': managed_stream.claim,
|
||||
'metadata': managed_stream.metadata,
|
||||
'protobuf': managed_stream.metadata_protobuf,
|
||||
'channel_claim_id': managed_stream.channel_claim_id,
|
||||
'channel_name': managed_stream.channel_name,
|
||||
'claim_name': managed_stream.claim_name,
|
||||
'content_fee': managed_stream.content_fee,
|
||||
'height': tx_height,
|
||||
'confirmations': confirmations,
|
||||
'timestamp': timestamp
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
if 'public_key' in encoded:
|
||||
encoded['public_key_id'] = self.ledger.public_key_to_address(
|
||||
unhexlify(encoded['public_key'])
|
||||
)
|
||||
return encoded
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
import logging.handlers
|
||||
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from torba import __version__ as torba_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,6 +19,7 @@ def get_platform() -> dict:
|
||||
"os_release": platform.release(),
|
||||
"os_system": os_system,
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"torba_version": torba_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
|
||||
@@ -195,14 +195,13 @@ class Stream(BaseClaim):
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
if 'media_type' in claim['source']:
|
||||
claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
|
||||
fee = claim.get('fee', {})
|
||||
if 'address' in fee:
|
||||
fee['address'] = self.fee.address
|
||||
if 'amount' in fee:
|
||||
fee['amount'] = str(self.fee.amount)
|
||||
stream_type = self.message.WhichOneof('type')
|
||||
if stream_type:
|
||||
claim['stream_type'] = stream_type
|
||||
return claim
|
||||
|
||||
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
|
||||
|
||||
@@ -140,8 +140,10 @@ types_map = {
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mkv': ('video/x-matroska', 'video'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
@@ -149,8 +151,7 @@ types_map = {
|
||||
'.prc': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.rtf': ('application/rtf', 'document'),
|
||||
'.xul': ('text/xul', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import NamedTuple, Tuple
|
||||
|
||||
|
||||
def _create_url_regex():
|
||||
# see https://spec.lbry.io/
|
||||
# see https://spec.lbry.com/
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
|
||||
@@ -166,6 +166,10 @@ class ManagedStream:
|
||||
def claim_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def claim(self) -> typing.Optional['Claim']:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
@@ -201,55 +205,6 @@ class ManagedStream:
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = self.full_path
|
||||
file_name = self.file_name
|
||||
download_directory = self.download_directory
|
||||
if self.full_path and self.output_file_exists:
|
||||
if self.written_bytes:
|
||||
written_bytes = self.written_bytes
|
||||
else:
|
||||
written_bytes = os.stat(self.full_path).st_size
|
||||
else:
|
||||
full_path = None
|
||||
file_name = None
|
||||
download_directory = None
|
||||
written_bytes = None
|
||||
return {
|
||||
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
|
||||
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
|
||||
or not self.saving.is_set()) or all(
|
||||
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not self.running,
|
||||
'stream_hash': self.stream_hash,
|
||||
'stream_name': self.descriptor.stream_name,
|
||||
'suggested_file_name': self.descriptor.suggested_file_name,
|
||||
'sd_hash': self.descriptor.sd_hash,
|
||||
'download_path': full_path,
|
||||
'mime_type': self.mime_type,
|
||||
'key': self.descriptor.key,
|
||||
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
'status': self.status,
|
||||
'claim_id': self.claim_id,
|
||||
'txid': self.txid,
|
||||
'nout': self.nout,
|
||||
'outpoint': self.outpoint,
|
||||
'metadata': self.metadata,
|
||||
'protobuf': self.metadata_protobuf,
|
||||
'channel_claim_id': self.channel_claim_id,
|
||||
'channel_name': self.channel_name,
|
||||
'claim_name': self.claim_name,
|
||||
'content_fee': self.content_fee
|
||||
}
|
||||
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
|
||||
@@ -10,13 +10,15 @@ if typing.TYPE_CHECKING:
|
||||
REFLECTOR_V1 = 0
|
||||
REFLECTOR_V2 = 1
|
||||
|
||||
MAX_RESPONSE_SIZE = 2000000
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.transport: typing.Optional[asyncio.WriteTransport] = None
|
||||
self.blob_manager = blob_manager
|
||||
self.descriptor = descriptor
|
||||
self.response_buff = b''
|
||||
@@ -37,11 +39,21 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
def data_received(self, data):
|
||||
if len(self.response_buff + (data or b'')) > MAX_RESPONSE_SIZE:
|
||||
log.warning("response message to large from reflector server: %i bytes",
|
||||
len(self.response_buff + (data or b'')))
|
||||
self.response_buff = b''
|
||||
self.transport.close()
|
||||
return
|
||||
self.response_buff += (data or b'')
|
||||
try:
|
||||
response = json.loads(data.decode())
|
||||
response = json.loads(self.response_buff.decode())
|
||||
self.response_buff = b''
|
||||
self.response_queue.put_nowait(response)
|
||||
except ValueError:
|
||||
self.transport.close()
|
||||
if not data:
|
||||
log.warning("got undecodable response from reflector server")
|
||||
self.response_buff = b''
|
||||
return
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict):
|
||||
|
||||
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager'):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -28,6 +28,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||
self.received = []
|
||||
self.incoming = asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
@@ -47,7 +48,15 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
def send_response(self, response: typing.Dict):
|
||||
self.transport.write(json.dumps(response).encode())
|
||||
def chunk_response(remaining: bytes):
|
||||
f = self.loop.create_future()
|
||||
f.add_done_callback(lambda _: self.transport.write(remaining[:self.chunk_size]))
|
||||
if len(remaining) > self.chunk_size:
|
||||
f.add_done_callback(lambda _: self.loop.call_soon(chunk_response, remaining[self.chunk_size:]))
|
||||
self.loop.call_soon(f.set_result, None)
|
||||
|
||||
response_bytes = json.dumps(response).encode()
|
||||
chunk_response(response_bytes)
|
||||
|
||||
async def handle_request(self, request: typing.Dict):
|
||||
if self.client_version is None:
|
||||
@@ -121,11 +130,12 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager'):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -133,7 +143,7 @@ class ReflectorServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(self.blob_manager),
|
||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
|
||||
@@ -58,7 +58,6 @@ class ExchangeRateManagerComponent(Component):
|
||||
|
||||
class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
timeout = 180
|
||||
LEDGER = lbrynet.wallet
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
+3
-3
@@ -75,7 +75,7 @@ def obfuscate(plain):
|
||||
return rot13(base64.b64encode(plain).decode())
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
try:
|
||||
@@ -102,7 +102,7 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
async def async_check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
||||
|
||||
|
||||
@@ -271,7 +271,7 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
|
||||
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
|
||||
@@ -59,7 +59,7 @@ class WalletDatabase(BaseDatabase):
|
||||
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and my_account is not None:
|
||||
|
||||
+152
-40
@@ -10,6 +10,7 @@ from torba.server.util import class_logger
|
||||
from torba.client.basedatabase import query, constraints_to_sql
|
||||
|
||||
from lbrynet.schema.url import URL, normalize_name
|
||||
from lbrynet.schema.mime_types import guess_stream_type
|
||||
from lbrynet.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.server.canonical import register_canonical_functions
|
||||
@@ -19,6 +20,18 @@ from lbrynet.wallet.server.trending import (
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
}
|
||||
STREAM_TYPES = {
|
||||
'video': 1,
|
||||
'audio': 2,
|
||||
'image': 3,
|
||||
'document': 4,
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr):
|
||||
@@ -87,9 +100,15 @@ class SQLDB:
|
||||
short_url text not null, -- normalized#shortest-unique-claim_id
|
||||
canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
|
||||
|
||||
claim_type integer,
|
||||
|
||||
-- streams
|
||||
stream_type text,
|
||||
media_type text,
|
||||
|
||||
-- claims which are channels
|
||||
is_channel bool not null,
|
||||
public_key_bytes bytes,
|
||||
public_key_hash bytes,
|
||||
claims_in_channel integer,
|
||||
|
||||
-- claims which are inside channels
|
||||
@@ -97,7 +116,7 @@ class SQLDB:
|
||||
channel_join integer, -- height at which claim got valid signature / joined channel
|
||||
signature bytes,
|
||||
signature_digest bytes,
|
||||
is_channel_signature_valid bool not null default false,
|
||||
signature_valid bool,
|
||||
|
||||
effective_amount integer not null default 0,
|
||||
support_amount integer not null default 0,
|
||||
@@ -107,16 +126,27 @@ class SQLDB:
|
||||
trending_global integer not null default 0
|
||||
);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id);
|
||||
create index if not exists claim_normalized_idx on claim (normalized);
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_channel_hash_idx on claim (channel_hash);
|
||||
create index if not exists claim_release_time_idx on claim (release_time);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_height_idx on claim (height);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_claim_type_idx on claim (claim_type);
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
|
||||
create index if not exists claim_effective_amount_idx on claim (effective_amount);
|
||||
create index if not exists claim_trending_group_idx on claim (trending_group);
|
||||
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
|
||||
create index if not exists claim_trending_local_idx on claim (trending_local);
|
||||
@@ -239,9 +269,11 @@ class SQLDB:
|
||||
'txo_hash': sqlite3.Binary(txo.ref.hash),
|
||||
'tx_position': tx.position,
|
||||
'amount': txo.amount,
|
||||
'is_channel': False,
|
||||
'timestamp': header['timestamp'],
|
||||
'height': tx.height,
|
||||
'claim_type': None,
|
||||
'stream_type': None,
|
||||
'media_type': None,
|
||||
'release_time': None,
|
||||
}
|
||||
claims.append(claim_record)
|
||||
@@ -253,10 +285,13 @@ class SQLDB:
|
||||
continue
|
||||
|
||||
if claim.is_stream:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['stream']
|
||||
claim_record['media_type'] = claim.stream.source.media_type
|
||||
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
|
||||
if claim.stream.release_time:
|
||||
claim_record['release_time'] = claim.stream.release_time
|
||||
elif claim.is_channel:
|
||||
claim_record['is_channel'] = True
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
|
||||
for tag in claim.message.tags:
|
||||
tags.append((tag, claim_hash, tx.height))
|
||||
@@ -277,15 +312,15 @@ class SQLDB:
|
||||
self.db.executemany("""
|
||||
INSERT INTO claim (
|
||||
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
|
||||
is_channel, timestamp, creation_timestamp, height, creation_height,
|
||||
release_time, activation_height, expiration_height, short_url)
|
||||
claim_type, media_type, stream_type, timestamp, creation_timestamp, height,
|
||||
creation_height, release_time, activation_height, expiration_height, short_url)
|
||||
VALUES (
|
||||
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
|
||||
:is_channel, :timestamp, :timestamp, :height, :height,
|
||||
:claim_type, :media_type, :stream_type, :timestamp, :timestamp, :height, :height,
|
||||
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
|
||||
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
|
||||
CASE WHEN :height >= 262974 THEN :height+2102400 ELSE :height+262974 END,
|
||||
:normalized||COALESCE(
|
||||
CASE WHEN :height >= 137181 THEN :height+2102400 ELSE :height+262974 END,
|
||||
:claim_name||COALESCE(
|
||||
(SELECT shortest_id(claim_id, :claim_id) FROM claim WHERE normalized = :normalized),
|
||||
'#'||substr(:claim_id, 1, 1)
|
||||
)
|
||||
@@ -296,7 +331,9 @@ class SQLDB:
|
||||
if claims:
|
||||
self.db.executemany("""
|
||||
UPDATE claim SET
|
||||
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height, timestamp=:timestamp,
|
||||
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
|
||||
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
|
||||
timestamp=:timestamp,
|
||||
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
|
||||
WHERE claim_hash=:claim_hash;
|
||||
""", claims)
|
||||
@@ -352,10 +389,12 @@ class SQLDB:
|
||||
'support', {'txo_hash__in': [sqlite3.Binary(txo_hash) for txo_hash in txo_hashes]}
|
||||
))
|
||||
|
||||
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims):
|
||||
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, timer):
|
||||
if not new_claims and not updated_claims and not spent_claims:
|
||||
return
|
||||
|
||||
sub_timer = timer.add_timer('segregate channels and signables')
|
||||
sub_timer.start()
|
||||
channels, new_channel_keys, signables = {}, {}, {}
|
||||
for txo in chain(new_claims, updated_claims):
|
||||
try:
|
||||
@@ -367,13 +406,19 @@ class SQLDB:
|
||||
new_channel_keys[txo.claim_hash] = claim.channel.public_key_bytes
|
||||
else:
|
||||
signables[txo.claim_hash] = txo
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('make list of channels we need to lookup')
|
||||
sub_timer.start()
|
||||
missing_channel_keys = set()
|
||||
for txo in signables.values():
|
||||
claim = txo.claim
|
||||
if claim.is_signed and claim.signing_channel_hash not in new_channel_keys:
|
||||
missing_channel_keys.add(claim.signing_channel_hash)
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('lookup missing channels')
|
||||
sub_timer.start()
|
||||
all_channel_keys = {}
|
||||
if new_channel_keys or missing_channel_keys:
|
||||
all_channel_keys = dict(self.execute(*query(
|
||||
@@ -383,7 +428,10 @@ class SQLDB:
|
||||
set(new_channel_keys) | missing_channel_keys
|
||||
]
|
||||
)))
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('prepare for updating claims')
|
||||
sub_timer.start()
|
||||
changed_channel_keys = {}
|
||||
for claim_hash, new_key in new_channel_keys.items():
|
||||
if claim_hash not in all_channel_keys or all_channel_keys[claim_hash] != new_key:
|
||||
@@ -399,16 +447,20 @@ class SQLDB:
|
||||
'channel_hash': None,
|
||||
'signature': None,
|
||||
'signature_digest': None,
|
||||
'is_channel_signature_valid': False
|
||||
'signature_valid': None
|
||||
}
|
||||
if claim.is_signed:
|
||||
update.update({
|
||||
'channel_hash': sqlite3.Binary(claim.signing_channel_hash),
|
||||
'signature': sqlite3.Binary(txo.get_encoded_signature()),
|
||||
'signature_digest': sqlite3.Binary(txo.get_signature_digest(self.ledger))
|
||||
'signature_digest': sqlite3.Binary(txo.get_signature_digest(self.ledger)),
|
||||
'signature_valid': 0
|
||||
})
|
||||
claim_updates.append(update)
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('find claims affected by a change in channel key')
|
||||
sub_timer.start()
|
||||
if changed_channel_keys:
|
||||
sql = f"""
|
||||
SELECT * FROM claim WHERE
|
||||
@@ -422,66 +474,91 @@ class SQLDB:
|
||||
'channel_hash': sqlite3.Binary(affected_claim['channel_hash']),
|
||||
'signature': sqlite3.Binary(affected_claim['signature']),
|
||||
'signature_digest': sqlite3.Binary(affected_claim['signature_digest']),
|
||||
'is_channel_signature_valid': False
|
||||
'signature_valid': 0
|
||||
})
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('verify signatures')
|
||||
sub_timer.start()
|
||||
for update in claim_updates:
|
||||
channel_pub_key = all_channel_keys.get(update['channel_hash'])
|
||||
if channel_pub_key and update['signature']:
|
||||
update['is_channel_signature_valid'] = Output.is_signature_valid(
|
||||
update['signature_valid'] = Output.is_signature_valid(
|
||||
bytes(update['signature']), bytes(update['signature_digest']), channel_pub_key
|
||||
)
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('update claims')
|
||||
sub_timer.start()
|
||||
if claim_updates:
|
||||
self.db.executemany(f"""
|
||||
UPDATE claim SET
|
||||
channel_hash=:channel_hash, signature=:signature, signature_digest=:signature_digest,
|
||||
is_channel_signature_valid=:is_channel_signature_valid,
|
||||
signature_valid=:signature_valid,
|
||||
channel_join=CASE
|
||||
WHEN is_channel_signature_valid AND :is_channel_signature_valid THEN channel_join
|
||||
WHEN :is_channel_signature_valid THEN {height}
|
||||
WHEN signature_valid=1 AND :signature_valid=1 THEN channel_join
|
||||
WHEN :signature_valid=1 THEN {height}
|
||||
END,
|
||||
canonical_url=CASE
|
||||
WHEN is_channel_signature_valid AND :is_channel_signature_valid THEN canonical_url
|
||||
WHEN :is_channel_signature_valid THEN
|
||||
WHEN signature_valid=1 AND :signature_valid=1 THEN canonical_url
|
||||
WHEN :signature_valid=1 THEN
|
||||
(SELECT short_url FROM claim WHERE claim_hash=:channel_hash)||'/'||
|
||||
normalized||COALESCE(
|
||||
claim_name||COALESCE(
|
||||
(SELECT shortest_id(other_claim.claim_id, claim.claim_id) FROM claim AS other_claim
|
||||
WHERE other_claim.normalized = claim.normalized AND
|
||||
WHERE other_claim.signature_valid = 1 AND
|
||||
other_claim.channel_hash = :channel_hash AND
|
||||
other_claim.is_channel_signature_valid = 1),
|
||||
other_claim.normalized = claim.normalized),
|
||||
'#'||substr(claim_id, 1, 1)
|
||||
)
|
||||
END
|
||||
WHERE claim_hash=:claim_hash;
|
||||
""", claim_updates)
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('update claims affected by spent channels')
|
||||
sub_timer.start()
|
||||
if spent_claims:
|
||||
self.execute(
|
||||
f"""
|
||||
UPDATE claim SET is_channel_signature_valid=0, channel_join=NULL, canonical_url=NULL
|
||||
UPDATE claim SET
|
||||
signature_valid=CASE WHEN signature IS NOT NULL THEN 0 END,
|
||||
channel_join=NULL, canonical_url=NULL
|
||||
WHERE channel_hash IN ({','.join('?' for _ in spent_claims)})
|
||||
""", [sqlite3.Binary(cid) for cid in spent_claims]
|
||||
)
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('update channels')
|
||||
sub_timer.start()
|
||||
if channels:
|
||||
self.db.executemany(
|
||||
"UPDATE claim SET public_key_bytes=:public_key_bytes WHERE claim_hash=:claim_hash", [{
|
||||
"""
|
||||
UPDATE claim SET
|
||||
public_key_bytes=:public_key_bytes,
|
||||
public_key_hash=:public_key_hash
|
||||
WHERE claim_hash=:claim_hash""", [{
|
||||
'claim_hash': sqlite3.Binary(claim_hash),
|
||||
'public_key_bytes': sqlite3.Binary(txo.claim.channel.public_key_bytes)
|
||||
'public_key_bytes': sqlite3.Binary(txo.claim.channel.public_key_bytes),
|
||||
'public_key_hash': sqlite3.Binary(
|
||||
self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(txo.claim.channel.public_key_bytes)))
|
||||
} for claim_hash, txo in channels.items()]
|
||||
)
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('update claims_in_channel counts')
|
||||
sub_timer.start()
|
||||
if all_channel_keys:
|
||||
self.db.executemany(f"""
|
||||
UPDATE claim SET
|
||||
claims_in_channel=(
|
||||
SELECT COUNT(*) FROM claim AS claim_in_channel
|
||||
WHERE claim_in_channel.channel_hash=claim.claim_hash AND
|
||||
claim_in_channel.is_channel_signature_valid
|
||||
WHERE claim_in_channel.signature_valid=1 AND
|
||||
claim_in_channel.channel_hash=claim.claim_hash
|
||||
)
|
||||
WHERE claim_hash = ?
|
||||
""", [(sqlite3.Binary(channel_hash),) for channel_hash in all_channel_keys.keys()])
|
||||
sub_timer.stop()
|
||||
|
||||
def _update_support_amount(self, claim_hashes):
|
||||
if claim_hashes:
|
||||
@@ -578,6 +655,11 @@ class SQLDB:
|
||||
r(self._update_effective_amount, height)
|
||||
r(self._perform_overtake, height, [], [])
|
||||
|
||||
def get_expiring(self, height):
|
||||
return self.execute(
|
||||
f"SELECT claim_hash, normalized FROM claim WHERE expiration_height = {height}"
|
||||
)
|
||||
|
||||
def advance_txs(self, height, all_txs, header, daemon_height, timer):
|
||||
insert_claims = []
|
||||
update_claims = []
|
||||
@@ -598,8 +680,8 @@ class SQLDB:
|
||||
)
|
||||
body_timer.start()
|
||||
delete_claim_hashes.update({r['claim_hash'] for r in spent_claims})
|
||||
delete_support_txo_hashes.update({r['txo_hash'] for r in spent_supports})
|
||||
deleted_claim_names.update({r['normalized'] for r in spent_claims})
|
||||
delete_support_txo_hashes.update({r['txo_hash'] for r in spent_supports})
|
||||
recalculate_claim_hashes.update({r['claim_hash'] for r in spent_supports})
|
||||
delete_others.update(spent_others)
|
||||
# Outputs
|
||||
@@ -617,23 +699,33 @@ class SQLDB:
|
||||
delete_claim_hashes.discard(claim_hash)
|
||||
delete_others.discard(output.ref.hash) # claim insertion and update occurring in the same block
|
||||
body_timer.stop()
|
||||
|
||||
skip_claim_timer = timer.add_timer('skip insertion of abandoned claims')
|
||||
skip_claim_timer.start()
|
||||
for new_claim in list(insert_claims):
|
||||
if new_claim.ref.hash in delete_others:
|
||||
insert_claims.remove(new_claim)
|
||||
skip_claim_timer.stop()
|
||||
|
||||
expire_timer = timer.add_timer('recording expired claims')
|
||||
expire_timer.start()
|
||||
for expired in self.get_expiring(height):
|
||||
delete_claim_hashes.add(expired['claim_hash'])
|
||||
deleted_claim_names.add(expired['normalized'])
|
||||
expire_timer.stop()
|
||||
|
||||
r = timer.run
|
||||
r(self.delete_claims, delete_claim_hashes)
|
||||
r(self.delete_supports, delete_support_txo_hashes)
|
||||
r(self.insert_claims, insert_claims, header)
|
||||
r(self.update_claims, update_claims, header)
|
||||
r(self.validate_channel_signatures, height, insert_claims, update_claims, delete_claim_hashes)
|
||||
r(self.validate_channel_signatures, height, insert_claims,
|
||||
update_claims, delete_claim_hashes, forward_timer=True)
|
||||
r(self.insert_supports, insert_supports)
|
||||
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
|
||||
r(calculate_trending, self.db, height, self.main.first_sync, daemon_height)
|
||||
|
||||
def get_claims(self, cols, **constraints):
|
||||
def get_claims(self, cols, join=True, **constraints):
|
||||
if 'order_by' in constraints:
|
||||
sql_order_by = []
|
||||
for order_by in constraints['order_by']:
|
||||
@@ -662,6 +754,7 @@ class SQLDB:
|
||||
|
||||
if constraints.pop('is_controlling', False):
|
||||
if {'sequence', 'amount_order'}.isdisjoint(constraints):
|
||||
join = True
|
||||
constraints['claimtrie.claim_hash__is_not_null'] = ''
|
||||
if 'sequence' in constraints:
|
||||
constraints['order_by'] = 'claim.activation_height ASC'
|
||||
@@ -682,6 +775,10 @@ class SQLDB:
|
||||
if 'name' in constraints:
|
||||
constraints['claim.normalized'] = normalize_name(constraints.pop('name'))
|
||||
|
||||
if 'public_key_id' in constraints:
|
||||
constraints['claim.public_key_hash'] = sqlite3.Binary(
|
||||
self.ledger.address_to_hash160(constraints.pop('public_key_id')))
|
||||
|
||||
if 'channel' in constraints:
|
||||
channel_url = constraints.pop('channel')
|
||||
match = self._resolve_one(channel_url)
|
||||
@@ -705,13 +802,27 @@ class SQLDB:
|
||||
tx_hash + struct.pack('<I', nout)
|
||||
)
|
||||
|
||||
if 'claim_type' in constraints:
|
||||
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
|
||||
if 'stream_types' in constraints:
|
||||
stream_types = constraints.pop('stream_types')
|
||||
if stream_types:
|
||||
constraints['claim.stream_type__in'] = [
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
]
|
||||
if 'media_types' in constraints:
|
||||
media_types = constraints.pop('media_types')
|
||||
if media_types:
|
||||
constraints['claim.media_type__in'] = media_types
|
||||
|
||||
_apply_constraints_for_array_attributes(constraints, 'tag')
|
||||
_apply_constraints_for_array_attributes(constraints, 'language')
|
||||
_apply_constraints_for_array_attributes(constraints, 'location')
|
||||
|
||||
select = f"SELECT {cols} FROM claim"
|
||||
|
||||
sql, values = query(
|
||||
f"""
|
||||
SELECT {cols} FROM claim
|
||||
select if not join else select+"""
|
||||
LEFT JOIN claimtrie USING (claim_hash)
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
|
||||
""", **constraints
|
||||
@@ -726,7 +837,7 @@ class SQLDB:
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = self.get_claims('count(*)', **constraints)
|
||||
count = self.get_claims('count(*)', join=False, **constraints)
|
||||
return count[0][0]
|
||||
|
||||
def _search(self, **constraints):
|
||||
@@ -734,7 +845,7 @@ class SQLDB:
|
||||
"""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.is_channel, claim.claims_in_channel,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
@@ -742,21 +853,22 @@ class SQLDB:
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
|
||||
channel.height AS channel_height, claim.is_channel_signature_valid
|
||||
channel.height AS channel_height, claim.signature_valid
|
||||
""", **constraints
|
||||
)
|
||||
|
||||
INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'release_time',
|
||||
'tx_position', 'channel_join', 'is_channel_signature_valid',
|
||||
'tx_position', 'channel_join', 'signature_valid',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed',
|
||||
'trending_local', 'trending_global',
|
||||
}
|
||||
|
||||
SEARCH_PARAMS = {
|
||||
'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids',
|
||||
'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids', 'public_key_id',
|
||||
'claim_type', 'stream_types', 'media_types',
|
||||
'any_tags', 'all_tags', 'not_tags',
|
||||
'any_locations', 'all_locations', 'not_locations',
|
||||
'any_languages', 'all_languages', 'not_languages',
|
||||
@@ -811,7 +923,7 @@ class SQLDB:
|
||||
else:
|
||||
query['order_by'] = ['^channel_join']
|
||||
query['channel_hash'] = channel['claim_hash']
|
||||
query['is_channel_signature_valid'] = 1
|
||||
query['signature_valid'] = 1
|
||||
elif set(query) == {'name'}:
|
||||
query['is_controlling'] = 1
|
||||
matches = self._search(**query, limit=1)
|
||||
|
||||
@@ -87,6 +87,13 @@ class Output(BaseOutput):
|
||||
return self.script.values['claim']
|
||||
raise ValueError('Only claim name and claim update have the claim payload.')
|
||||
|
||||
@property
|
||||
def can_decode_claim(self):
|
||||
try:
|
||||
return self.claim
|
||||
except:
|
||||
return False
|
||||
|
||||
@property
|
||||
def permanent_url(self) -> str:
|
||||
if self.script.is_claim_involved:
|
||||
|
||||
+1
-1
@@ -24,6 +24,6 @@ theme:
|
||||
- type: 'telegram'
|
||||
link: 'https://t.me/lbryofficial'
|
||||
- type: 'discord'
|
||||
link: 'https://chat.lbry.io'
|
||||
link: 'https://chat.lbry.com'
|
||||
- type: 'reddit'
|
||||
link: 'https://reddit.com/r/lbry'
|
||||
@@ -1,5 +1,7 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG TORBA_VERSION=master
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
@@ -15,7 +17,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools wheel
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
|
||||
RUN python3.7 -m pip install -e .
|
||||
|
||||
# Orchstr8 API
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG TORBA_VERSION=master
|
||||
|
||||
# create an unprivileged user
|
||||
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
|
||||
@@ -23,7 +24,7 @@ WORKDIR /home/$user
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
|
||||
# get torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
|
||||
|
||||
# get uvloop
|
||||
RUN python3.7 -m pip install --user uvloop
|
||||
|
||||
@@ -47,7 +47,7 @@ async def main(blob_hash: str, url: str):
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
url = 'reflector.lbry.io:5567'
|
||||
url = 'reflector.lbry.com:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
asyncio.run(main(sys.argv[1], url))
|
||||
@@ -26,7 +26,7 @@ def extract_uris(response):
|
||||
async def get_frontpage_uris():
|
||||
session = aiohttp.ClientSession()
|
||||
try:
|
||||
response = await session.get("https://api.lbry.io/file/list_homepage", timeout=10.0)
|
||||
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
|
||||
if response.status != 200:
|
||||
print("API returned non 200 code!!")
|
||||
return
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
set -x
|
||||
TORBA=$1
|
||||
# must also be updated in travis.yml
|
||||
TORBA_VERSION=master
|
||||
rm -rf /tmp/.wine-*
|
||||
|
||||
apt-get -qq update
|
||||
apt-get -qq install -y git
|
||||
|
||||
pip install setuptools_scm
|
||||
|
||||
cd lbry
|
||||
@@ -12,8 +14,13 @@ cd lbry
|
||||
wget -Onetifaces-0.10.7-cp37-cp37m-win32.whl https://ci.appveyor.com/api/buildjobs/6hworunifsymrhp2/artifacts/dist%2Fnetifaces-0.10.7-cp37-cp37m-win32.whl
|
||||
pip install netifaces-0.10.7-cp37-cp37m-win32.whl
|
||||
|
||||
pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
git clone --depth=1 --single-branch --branch ${TORBA_VERSION} https://github.com/lbryio/torba.git
|
||||
cd torba
|
||||
pip install .
|
||||
cd ..
|
||||
rm -rf torba
|
||||
|
||||
pip show torba
|
||||
pip install -e .
|
||||
pip install pywin32
|
||||
|
||||
|
||||
@@ -9,8 +9,8 @@ setup(
|
||||
name=__name__,
|
||||
version=__version__,
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
author_email="hello@lbry.com",
|
||||
url="https://lbry.com",
|
||||
description="A decentralized media library and marketplace",
|
||||
long_description=open(README_PATH, encoding='utf-8').read(),
|
||||
long_description_content_type="text/markdown",
|
||||
@@ -23,9 +23,9 @@ setup(
|
||||
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
|
||||
},
|
||||
install_requires=[
|
||||
'torba==0.5.4',
|
||||
'torba==0.5.5',
|
||||
'aiohttp==3.5.4',
|
||||
'aioupnp',
|
||||
'aioupnp==0.0.12',
|
||||
'appdirs==1.4.3',
|
||||
'certifi>=2018.11.29',
|
||||
'colorama==0.3.7',
|
||||
|
||||
@@ -8,12 +8,61 @@ from urllib.request import urlopen
|
||||
from torba.client.errors import InsufficientFundsError
|
||||
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClaimSearchCommand(CommandTestCase):
|
||||
class ClaimTestCase(CommandTestCase):
|
||||
|
||||
files_directory = os.path.join(os.path.dirname(__file__), 'files')
|
||||
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
|
||||
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
|
||||
|
||||
def setUp(self):
|
||||
if not os.path.exists(self.video_file_name):
|
||||
if not os.path.exists(self.files_directory):
|
||||
os.mkdir(self.files_directory)
|
||||
log.info(f'downloading test video from {self.video_file_name}')
|
||||
with urlopen(self.video_file_url) as response, \
|
||||
open(self.video_file_name, 'wb') as video_file:
|
||||
video_file.write(response.read())
|
||||
|
||||
async def image_stream_create(self, name='blank-image', bid='1.0', confirm=True):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as file:
|
||||
file.write(unhexlify(
|
||||
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
|
||||
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
|
||||
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
|
||||
b'9454e44ae426082'
|
||||
))
|
||||
file.flush()
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
name, bid, file_path=file.name
|
||||
)
|
||||
)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(tx)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tx)
|
||||
return tx
|
||||
|
||||
async def video_stream_create(self, name='chrome', bid='1.0', confirm=True):
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
name, bid, file_path=self.video_file_name
|
||||
)
|
||||
)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(tx)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tx)
|
||||
return tx
|
||||
|
||||
|
||||
class ClaimSearchCommand(ClaimTestCase):
|
||||
|
||||
async def create_channel(self):
|
||||
self.channel = await self.channel_create('@abc', '1.0')
|
||||
@@ -50,8 +99,10 @@ class ClaimSearchCommand(CommandTestCase):
|
||||
|
||||
async def test_basic_claim_search(self):
|
||||
await self.create_channel()
|
||||
channel_txo = self.channel['outputs'][0]
|
||||
channel2 = await self.channel_create('@abc', '0.1', allow_duplicate_name=True)
|
||||
channel_id2 = channel2['outputs'][0]['claim_id']
|
||||
channel_txo2 = channel2['outputs'][0]
|
||||
channel_id2 = channel_txo2['claim_id']
|
||||
|
||||
# finding a channel
|
||||
await self.assertFindsClaims([channel2, self.channel], name='@abc')
|
||||
@@ -60,6 +111,10 @@ class ClaimSearchCommand(CommandTestCase):
|
||||
await self.assertFindsClaim(self.channel, txid=self.channel['txid'], nout=0)
|
||||
await self.assertFindsClaim(channel2, claim_id=channel_id2)
|
||||
await self.assertFindsClaim(channel2, txid=channel2['txid'], nout=0)
|
||||
await self.assertFindsClaim(
|
||||
channel2, public_key_id=channel_txo2['value']['public_key_id'])
|
||||
await self.assertFindsClaim(
|
||||
self.channel, public_key_id=channel_txo['value']['public_key_id'])
|
||||
|
||||
signed = await self.stream_create('on-channel-claim', '0.001', channel_id=self.channel_id)
|
||||
signed2 = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel_id2,
|
||||
@@ -86,8 +141,20 @@ class ClaimSearchCommand(CommandTestCase):
|
||||
await self.channel_abandon(claim_id=self.channel_id)
|
||||
await self.assertFindsClaims([], channel_ids=[self.channel_id], valid_channel_signatures=True)
|
||||
await self.assertFindsClaims([signed2], channel_ids=[channel_id2], valid_channel_signatures=True)
|
||||
# pass `invalid_channel_signatures=False` to catch a bug in argument processing
|
||||
await self.assertFindsClaims([signed2], channel_ids=[channel_id2, self.channel_id],
|
||||
valid_channel_signatures=True)
|
||||
valid_channel_signatures=True, invalid_channel_signatures=False)
|
||||
# invalid signature still returns channel_id
|
||||
self.ledger._tx_cache.clear()
|
||||
invalid_claims = await self.claim_search(invalid_channel_signatures=True)
|
||||
self.assertEqual(3, len(invalid_claims))
|
||||
self.assertTrue(all([not c['is_channel_signature_valid'] for c in invalid_claims]))
|
||||
self.assertEqual({'channel_id': self.channel_id}, invalid_claims[0]['signing_channel'])
|
||||
|
||||
valid_claims = await self.claim_search(valid_channel_signatures=True)
|
||||
self.assertEqual(1, len(valid_claims))
|
||||
self.assertTrue(all([c['is_channel_signature_valid'] for c in valid_claims]))
|
||||
self.assertEqual('@abc', valid_claims[0]['signing_channel']['name'])
|
||||
|
||||
# abandoned stream won't show up for streams in channel search
|
||||
await self.stream_abandon(txid=signed2['txid'], nout=0)
|
||||
@@ -164,6 +231,37 @@ class ClaimSearchCommand(CommandTestCase):
|
||||
|
||||
await self.assertFindsClaims(claims, order_by=["^name"])
|
||||
|
||||
async def test_claim_type_and_media_type_search(self):
|
||||
# create an invalid/unknown claim
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
tx = await Transaction.claim_create(
|
||||
'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
octet = await self.stream_create()
|
||||
video = await self.video_stream_create()
|
||||
image = await self.image_stream_create()
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
await self.assertFindsClaims([video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
|
||||
|
||||
|
||||
class ChannelCommands(CommandTestCase):
|
||||
|
||||
@@ -241,19 +339,29 @@ class ChannelCommands(CommandTestCase):
|
||||
# create new channel with all fields set
|
||||
tx = await self.out(self.channel_create('@bigchannel', **values))
|
||||
channel = tx['outputs'][0]['value']
|
||||
self.assertEqual(channel, {'public_key': channel['public_key'], **fixed_values})
|
||||
self.assertEqual(channel, {
|
||||
'public_key': channel['public_key'],
|
||||
'public_key_id': channel['public_key_id'],
|
||||
**fixed_values
|
||||
})
|
||||
|
||||
# create channel with nothing set
|
||||
tx = await self.out(self.channel_create('@lightchannel'))
|
||||
channel = tx['outputs'][0]['value']
|
||||
self.assertEqual(channel, {'public_key': channel['public_key']})
|
||||
self.assertEqual(
|
||||
channel, {'public_key': channel['public_key'], 'public_key_id': channel['public_key_id']})
|
||||
|
||||
# create channel with just a featured claim
|
||||
tx = await self.out(self.channel_create('@featurechannel', featured='beef'))
|
||||
txo = tx['outputs'][0]
|
||||
claim_id, channel = txo['claim_id'], txo['value']
|
||||
fixed_values['public_key'] = channel['public_key']
|
||||
self.assertEqual(channel, {'public_key': fixed_values['public_key'], 'featured': ['beef']})
|
||||
fixed_values['public_key_id'] = channel['public_key_id']
|
||||
self.assertEqual(channel, {
|
||||
'public_key': fixed_values['public_key'],
|
||||
'public_key_id': fixed_values['public_key_id'],
|
||||
'featured': ['beef']
|
||||
})
|
||||
|
||||
# update channel "@featurechannel" setting all fields
|
||||
tx = await self.out(self.channel_update(claim_id, **values))
|
||||
@@ -274,9 +382,10 @@ class ChannelCommands(CommandTestCase):
|
||||
|
||||
# replace mode (clears everything except public_key)
|
||||
tx = await self.out(self.channel_update(claim_id, replace=True, title='foo', email='new@email.com'))
|
||||
self.assertEqual(
|
||||
tx['outputs'][0]['value'],
|
||||
{'public_key': channel['public_key'], 'title': 'foo', 'email': 'new@email.com'}
|
||||
self.assertEqual(tx['outputs'][0]['value'], {
|
||||
'public_key': channel['public_key'],
|
||||
'public_key_id': channel['public_key_id'],
|
||||
'title': 'foo', 'email': 'new@email.com'}
|
||||
)
|
||||
|
||||
# send channel to someone else
|
||||
@@ -353,20 +462,7 @@ class ChannelCommands(CommandTestCase):
|
||||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
|
||||
|
||||
class StreamCommands(CommandTestCase):
|
||||
|
||||
files_directory = os.path.join(os.path.dirname(__file__), 'files')
|
||||
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
|
||||
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
|
||||
|
||||
def setUp(self):
|
||||
if not os.path.exists(self.video_file_name):
|
||||
if not os.path.exists(self.files_directory):
|
||||
os.mkdir(self.files_directory)
|
||||
log.info(f'downloading test video from {self.video_file_name}')
|
||||
with urlopen(self.video_file_url) as response,\
|
||||
open(self.video_file_name, 'wb') as video_file:
|
||||
video_file.write(response.read())
|
||||
class StreamCommands(ClaimTestCase):
|
||||
|
||||
async def test_create_stream_names(self):
|
||||
# claim new name
|
||||
@@ -532,6 +628,7 @@ class StreamCommands(CommandTestCase):
|
||||
}]
|
||||
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
|
||||
fixed_values['release_time'] = str(values['release_time'])
|
||||
fixed_values['stream_type'] = 'binary'
|
||||
fixed_values['source'] = {
|
||||
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
|
||||
'media_type': 'application/octet-stream',
|
||||
@@ -555,6 +652,7 @@ class StreamCommands(CommandTestCase):
|
||||
stream = tx['outputs'][0]['value']
|
||||
self.assertEqual(
|
||||
stream, {
|
||||
'stream_type': 'binary',
|
||||
'source': {
|
||||
'size': '3',
|
||||
'media_type': 'application/octet-stream',
|
||||
@@ -573,6 +671,7 @@ class StreamCommands(CommandTestCase):
|
||||
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
|
||||
self.assertEqual(
|
||||
stream, {
|
||||
'stream_type': 'binary',
|
||||
'source': {
|
||||
'size': '3',
|
||||
'media_type': 'application/octet-stream',
|
||||
@@ -645,44 +744,26 @@ class StreamCommands(CommandTestCase):
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
|
||||
|
||||
async def test_automatic_type_and_metadata_detection_for_image(self):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as file:
|
||||
file.write(unhexlify(
|
||||
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
|
||||
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
|
||||
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
|
||||
b'9454e44ae426082'
|
||||
))
|
||||
file.flush()
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'blank-image', '1.0', file_path=file.name
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '99',
|
||||
'name': os.path.basename(file.name),
|
||||
'media_type': 'image/png',
|
||||
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'image',
|
||||
'image': {
|
||||
'width': 5,
|
||||
'height': 7
|
||||
}
|
||||
txo = (await self.image_stream_create())['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '99',
|
||||
'name': txo['value']['source']['name'],
|
||||
'media_type': 'image/png',
|
||||
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'image',
|
||||
'image': {
|
||||
'width': 5,
|
||||
'height': 7
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
async def test_automatic_type_and_metadata_detection_for_video(self):
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'chrome', '1.0', file_path=self.video_file_name
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
txo = (await self.video_stream_create())['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
|
||||
@@ -1,22 +1,16 @@
|
||||
import logging
|
||||
|
||||
import typing
|
||||
import random
|
||||
import asyncio
|
||||
from aiohttp import web
|
||||
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
|
||||
class FakedCommentServer:
|
||||
|
||||
ERRORS = {
|
||||
'INVALID_URI': {'code': 1, 'message': 'Invalid claim URI'},
|
||||
'INVALID_PARAMS': {'code': -32602, 'message': 'Invalid parameters'},
|
||||
'INTERNAL': {'code': -32603, 'message': 'An internal error'},
|
||||
'UNKNOWN': {'code': -1, 'message': 'An unknown or very miscellaneous error'},
|
||||
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
|
||||
}
|
||||
|
||||
def __init__(self, port=2903):
|
||||
@@ -25,45 +19,21 @@ class FakedCommentServer:
|
||||
self.app.add_routes([web.post('/api', self.api)])
|
||||
self.runner = None
|
||||
self.server = None
|
||||
self.comments = []
|
||||
self.comment_id = 0
|
||||
|
||||
def get_claim_comments(self, uri: str, better_keys: bool) -> typing.Union[dict, list, None]:
|
||||
if not uri.startswith('lbry://'): # Very basic error case
|
||||
return {'error': self.ERRORS['INVALID_URI']}
|
||||
return [self.get_comment(i) for i in range(75)]
|
||||
def create_comment(self, **comment):
|
||||
self.comment_id += 1
|
||||
comment['comment_id'] = self.comment_id
|
||||
self.comments.append(comment)
|
||||
return comment
|
||||
|
||||
def get_comment(self, comment_id: int, parent_id: int = None) -> dict:
|
||||
return {
|
||||
'comment_id': comment_id,
|
||||
'parent_id': parent_id,
|
||||
'author': f'Person{comment_id}',
|
||||
'message': f'comment {comment_id}',
|
||||
'claim_id': random.randint(1, 2**16),
|
||||
'time_posted': random.randint(2**16, 2**32 - 1),
|
||||
'upvotes': random.randint(0, 9999), 'downvotes': random.randint(0, 9999)
|
||||
}
|
||||
|
||||
def comment(self, uri: str, poster: str, message: str) -> typing.Union[int, dict, None]:
|
||||
if not uri.startswith('lbry://'):
|
||||
return {'error': self.ERRORS['INVALID_URI']}
|
||||
return random.randint(1, 9999)
|
||||
|
||||
def reply(self, parent_id: int, poster: str, message: str) -> dict:
|
||||
if 2 <= len(message) <= 2000 and 2 <= len(poster) <= 127 and parent_id > 0:
|
||||
return random.randint(parent_id + 1, 2**32 - 1)
|
||||
return {'error': self.ERRORS['INVALID_PARAMS']}
|
||||
|
||||
def get_comment_data(self, comm_index: int, better_keys: bool = False) -> typing.Union[dict, None]:
|
||||
return self.get_comment(comm_index)
|
||||
|
||||
def get_comment_replies(self, comm_index: int) -> typing.Union[list, None]:
|
||||
return [random.randint(comm_index, comm_index+250) for _ in range(75)]
|
||||
def get_claim_comments(self, page=1, page_size=50, **kwargs):
|
||||
return self.comments[:page_size]
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
'get_comment_data': get_comment_data,
|
||||
'get_comment_replies': get_comment_replies,
|
||||
'comment': comment,
|
||||
'reply': reply
|
||||
'create_comment': create_comment,
|
||||
}
|
||||
|
||||
def process_json(self, body) -> dict:
|
||||
@@ -71,32 +41,21 @@ class FakedCommentServer:
|
||||
if body['method'] in self.methods:
|
||||
params = body.get('params', {})
|
||||
result = self.methods[body['method']](self, **params)
|
||||
if type(result) is dict and 'error' in result:
|
||||
response['error'] = result['error']
|
||||
else:
|
||||
response['result'] = result
|
||||
response['result'] = result
|
||||
else:
|
||||
response['error'] = self.ERRORS['UNKNOWN']
|
||||
response['error'] = self.ERRORS['INVALID_METHOD']
|
||||
return response
|
||||
|
||||
async def _start(self):
|
||||
async def start(self):
|
||||
self.runner = web.AppRunner(self.app)
|
||||
await self.runner.setup()
|
||||
self.server = web.TCPSite(self.runner, 'localhost', self.port)
|
||||
await self.server.start()
|
||||
|
||||
async def _stop(self):
|
||||
async def stop(self):
|
||||
await self.runner.shutdown()
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def run(self, max_timeout=3600):
|
||||
try:
|
||||
await self._start()
|
||||
await asyncio.sleep(max_timeout)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
await self._stop()
|
||||
|
||||
async def api(self, request):
|
||||
body = await request.json()
|
||||
if type(body) is list or type(body) is dict:
|
||||
@@ -106,98 +65,41 @@ class FakedCommentServer:
|
||||
response = self.process_json(body)
|
||||
return web.json_response(response)
|
||||
else:
|
||||
return web.json_response({'error': self.ERRORS['UNKNOWN']})
|
||||
raise TypeError('invalid type passed')
|
||||
|
||||
|
||||
class CommentCommands(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.daemon.conf.comment_server = 'http://localhost:2903/api'
|
||||
self.server = FakedCommentServer(2903)
|
||||
self.server_task = asyncio.create_task(self.server.run(self.timeout))
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.server_task.cancel()
|
||||
if not self.server_task.cancelled():
|
||||
await self.server_task
|
||||
self.comment_server = FakedCommentServer(2903)
|
||||
await self.comment_server.start()
|
||||
self.addCleanup(self.comment_server.stop)
|
||||
|
||||
async def test_comment_create(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001', data=b'loool')
|
||||
self.assertIn('outputs', claim)
|
||||
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
|
||||
stream = (await self.stream_create())['outputs'][0]
|
||||
|
||||
self.assertEqual(0, len(await self.daemon.jsonrpc_comment_list(stream['claim_id'])))
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=claim['outputs'][0]['claim_id'],
|
||||
channel_id='Jimmy Buffett',
|
||||
message="It's 5 O'Clock Somewhere"
|
||||
claim_id=stream['claim_id'],
|
||||
channel_id=channel['claim_id'],
|
||||
comment="It's 5 O'Clock Somewhere"
|
||||
)
|
||||
self.assertIs(type(comment), dict, msg=f"Response type ({type(comment)})is not dict: {comment}")
|
||||
self.assertIn('message', comment, msg=f"Response {comment} doesn't contain message")
|
||||
self.assertIn('author', comment)
|
||||
comments = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
|
||||
self.assertEqual(1, len(comments))
|
||||
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
|
||||
self.assertEqual(stream['claim_id'], comments[0]['claim_id'])
|
||||
|
||||
async def test_comment_create_reply(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
reply = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=claim['outputs'][0]['claim_id'],
|
||||
channel_id='Jimmy Buffett',
|
||||
message='Let\'s all go to Margaritaville',
|
||||
parent_comment_id=42
|
||||
channel2 = (await self.channel_create('@BuffettJimmy'))['outputs'][0]
|
||||
await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=stream['claim_id'],
|
||||
channel_name=channel2['name'],
|
||||
comment='Let\'s all go to Margaritaville',
|
||||
parent_id=comments[0]['comment_id']
|
||||
)
|
||||
self.assertIs(type(reply), dict, msg=f'Response {type(reply)} is not dict\nResponse: {reply}')
|
||||
self.assertIn('author', reply)
|
||||
|
||||
async def test_comment_list_root_level(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
claim_id = claim['outputs'][0]['claim_id']
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
self.assertIsNotNone(type(comments))
|
||||
self.assertIs(type(comments), dict)
|
||||
self.assertIn('comments', comments, f"'comments' field was not found in returned dict: {comments}")
|
||||
self.assertIs(type(comments['comments']), list, msg=f'comment_list: {comments}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50)
|
||||
self.assertIsNotNone(comments)
|
||||
self.assertIs(type(comments), dict)
|
||||
self.assertIn('comments', comments, f"'comments' field was not found in returned dict: {comments}")
|
||||
comment_list = comments['comments']
|
||||
self.assertEqual(len(comment_list), 50, msg=f'comment_list incorrect size {len(comment_list)}: {comment_list}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50, page=2)
|
||||
self.assertEqual(len(comments['comments']), 25, msg=f'comment list page 2: {comments["comments"]}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50, page=3)
|
||||
self.assertEqual(len(comments['comments']), 0, msg=f'comment list is non-zero: {comments["comments"]}')
|
||||
|
||||
async def test_comment_list_replies(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
claim_id = claim['outputs'][0]['claim_id']
|
||||
replies = await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=23)
|
||||
self.assertIsInstance(replies['comments'], list, msg=f'Invalid type: {replies["comments"]} should be list')
|
||||
self.assertGreater(len(replies['comments']), 0, msg='Returned replies are empty')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=25, page_size=50))['comments']
|
||||
self.assertEqual(len(replies), 50, f'Replies invalid length ({len(replies)})')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=67,
|
||||
page_size=23, page=5))['comments']
|
||||
self.assertEqual(len(replies), 0, f'replies {replies} not 23: {len(replies)}')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=79,
|
||||
page_size=60, page=2))['comments']
|
||||
self.assertEqual(len(replies), 15, f'Size of replies is incorrect, should be 15: {replies}')
|
||||
|
||||
async def test_comment_list_flatness_flatness_LA(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
claim_id = claim['outputs'][0]['claim_id']
|
||||
replies = await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=23, flat=True)
|
||||
self.assertIsInstance(replies['comments'], list, msg=f'Invalid type: {replies["comments"]} should be list')
|
||||
self.assertGreater(len(replies['comments']), 0, msg='Returned replies are empty')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=25, flat=True,
|
||||
max_replies_shown=0, page_size=50))['comments']
|
||||
self.assertEqual(len(replies), 50, f'Replies invalid length ({len(replies)})')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=67,
|
||||
flat=True, page_size=23, page=5))['comments']
|
||||
self.assertEqual(len(replies), 0, f'replies {replies} not 23: {len(replies)}')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=79,
|
||||
page_size=60, page=2))['comments']
|
||||
self.assertGreaterEqual(len(replies), 15, f'Size of replies is incorrect, should be 15: {replies}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
|
||||
self.assertEqual(2, len(comments))
|
||||
self.assertEqual(comments[1]['channel_id'], channel2['claim_id'])
|
||||
self.assertEqual(comments[0]['comment_id'], comments[1]['parent_id'])
|
||||
@@ -134,7 +134,6 @@ class ResolveCommand(CommandTestCase):
|
||||
channel_id = self.get_claim_id(
|
||||
await self.channel_create('@abc', '1.1', allow_duplicate_name=True))
|
||||
await self.assertResolvesToClaimId(f'@abc', channel_id)
|
||||
await self.assertResolvesToClaimId(f'@abc#{channel_id[0]}', channel_id)
|
||||
await self.assertResolvesToClaimId(f'@abc#{channel_id[:10]}', channel_id)
|
||||
await self.assertResolvesToClaimId(f'@abc#{channel_id}', channel_id)
|
||||
channel = (await self.claim_search(claim_id=channel_id))[0]
|
||||
@@ -168,6 +167,7 @@ class ResolveCommand(CommandTestCase):
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
|
||||
orphan_claim = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel['claim_id'])
|
||||
abandoned_channel_id = channel['claim_id']
|
||||
await self.channel_abandon(txid=channel['txid'], nout=0)
|
||||
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
|
||||
orphan_claim_id = orphan_claim['outputs'][0]['claim_id']
|
||||
@@ -178,11 +178,13 @@ class ResolveCommand(CommandTestCase):
|
||||
self.assertEqual(response, {
|
||||
'lbry://@abc/on-channel-claim': {'error': 'lbry://@abc/on-channel-claim did not resolve to a claim'}
|
||||
})
|
||||
response = await self.resolve('lbry://on-channel-claim')
|
||||
self.assertNotIn('is_channel_signature_valid', response['lbry://on-channel-claim'])
|
||||
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
|
||||
self.assertFalse(response['is_channel_signature_valid'])
|
||||
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
|
||||
response = await self.resolve(direct_uri)
|
||||
self.assertNotIn('is_channel_signature_valid', response[direct_uri])
|
||||
response = (await self.resolve(direct_uri))[direct_uri]
|
||||
self.assertFalse(response['is_channel_signature_valid'])
|
||||
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
|
||||
await self.stream_abandon(claim_id=orphan_claim_id)
|
||||
|
||||
uri = 'lbry://@abc/on-channel-claim'
|
||||
|
||||
@@ -41,8 +41,8 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
|
||||
self.stream = await self.stream_manager.create_stream(file_path)
|
||||
|
||||
async def test_reflect_stream(self):
|
||||
reflector = ReflectorServer(self.server_blob_manager)
|
||||
async def _test_reflect_stream(self, response_chunk_size):
|
||||
reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
@@ -63,6 +63,12 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
|
||||
self.assertListEqual(sent, [])
|
||||
|
||||
async def test_reflect_stream(self):
|
||||
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=50), 3, loop=self.loop)
|
||||
|
||||
async def test_reflect_stream_small_response_chunks(self):
|
||||
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=30), 3, loop=self.loop)
|
||||
|
||||
async def test_announces(self):
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
|
||||
|
||||
@@ -4,7 +4,7 @@ import types
|
||||
import tempfile
|
||||
import unittest
|
||||
import argparse
|
||||
from lbrynet.conf import Config, BaseConfig, String, Integer, Toggle, Servers, NOT_SET
|
||||
from lbrynet.conf import Config, BaseConfig, String, Integer, Toggle, Servers, Strings, NOT_SET
|
||||
from lbrynet.error import InvalidCurrencyError
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ class TestConfig(BaseConfig):
|
||||
test_false_toggle = Toggle('toggle help', False)
|
||||
test_true_toggle = Toggle('toggle help', True)
|
||||
servers = Servers('servers help', [('localhost', 80)])
|
||||
strings = Strings('cheese', ['string'])
|
||||
|
||||
|
||||
class ConfigurationTests(unittest.TestCase):
|
||||
@@ -54,6 +55,7 @@ class ConfigurationTests(unittest.TestCase):
|
||||
self.assertTrue(c.test_true_toggle)
|
||||
self.assertFalse(c.test_false_toggle)
|
||||
self.assertEqual(c.servers, [('localhost', 80)])
|
||||
self.assertEqual(c.strings, ['string'])
|
||||
|
||||
args = parser.parse_args(['--test-str', 'blah'])
|
||||
c = TestConfig.create_from_arguments(args)
|
||||
@@ -76,10 +78,14 @@ class ConfigurationTests(unittest.TestCase):
|
||||
self.assertFalse(c.test_true_toggle)
|
||||
self.assertFalse(c.test_false_toggle)
|
||||
|
||||
args = parser.parse_args(['--servers', 'localhost:1', '192.168.0.1:2'])
|
||||
args = parser.parse_args(['--servers=localhost:1', '--servers=192.168.0.1:2'])
|
||||
c = TestConfig.create_from_arguments(args)
|
||||
self.assertEqual(c.servers, [('localhost', 1), ('192.168.0.1', 2)])
|
||||
|
||||
args = parser.parse_args(['--strings=cheddar', '--strings=mozzarella'])
|
||||
c = TestConfig.create_from_arguments(args)
|
||||
self.assertEqual(c.strings, ['cheddar', 'mozzarella'])
|
||||
|
||||
def test_environment(self):
|
||||
c = TestConfig()
|
||||
self.assertEqual(c.test_str, 'the default')
|
||||
|
||||
@@ -140,7 +140,7 @@ class TestSQLDB(unittest.TestCase):
|
||||
return [otx[0].tx.outputs[0] for otx in txs]
|
||||
|
||||
def state(self, controlling=None, active=None, accepted=None):
|
||||
self.assertEqual(controlling or [], self.get_controlling())
|
||||
self.assertEqual(controlling, self.get_controlling())
|
||||
self.assertEqual(active or [], self.get_active())
|
||||
self.assertEqual(accepted or [], self.get_accepted())
|
||||
|
||||
@@ -288,6 +288,28 @@ class TestSQLDB(unittest.TestCase):
|
||||
accepted=[]
|
||||
)
|
||||
|
||||
def test_winning_claim_expires_and_another_takes_over(self):
|
||||
advance, state = self.advance, self.state
|
||||
advance(10, [self.get_stream('Claim A', 11*COIN)])
|
||||
advance(20, [self.get_stream('Claim B', 10*COIN)])
|
||||
state(
|
||||
controlling=('Claim A', 11*COIN, 11*COIN, 10),
|
||||
active=[('Claim B', 10*COIN, 10*COIN, 20)],
|
||||
accepted=[]
|
||||
)
|
||||
advance(262984, [])
|
||||
state(
|
||||
controlling=('Claim B', 10*COIN, 10*COIN, 20),
|
||||
active=[],
|
||||
accepted=[]
|
||||
)
|
||||
advance(262994, [])
|
||||
state(
|
||||
controlling=None,
|
||||
active=[],
|
||||
accepted=[]
|
||||
)
|
||||
|
||||
def test_trending(self):
|
||||
advance, state = self.advance, self.state
|
||||
no_trend = self.get_stream('Claim A', COIN)
|
||||
|
||||
Reference in new issue
Block a user