Compare commits

...
5 changed files with 112 additions and 68 deletions

No files matched your search

+70 -9
View File
@@ -1,3 +1,4 @@
import os
import logging
from decimal import Decimal
from binascii import hexlify, unhexlify
@@ -93,10 +94,16 @@ def encode_file_doc():
'txid': '(str) None if claim is not found else the transaction id',
'nout': '(int) None if claim is not found else the transaction output index',
'outpoint': '(str) None if claim is not found else the tx and output',
'claim': '(Claim) None if claim is not found',
'metadata': '(dict) None if claim is not found else the claim metadata',
'protobuf': '(str) hex encoded Stream protobuf',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name'
'claim_name': '(str) None if claim is not found else the claim name',
'content_fee': '(Transaction) None if content claim did not have a fee',
'height': '(int) height of the claim',
'confirmations': '(int) number of blocks since the claim was confirmed',
'timestamp': '(int) timestamp of the block containing the claim'
}
@@ -218,15 +225,69 @@ class JSONResponseEncoder(JSONEncoder):
return result
def encode_file(self, managed_stream):
file = managed_stream.as_dict()
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
file.update({
full_path = managed_stream.full_path
file_name = managed_stream.file_name
download_directory = managed_stream.download_directory
if managed_stream.full_path and managed_stream.output_file_exists:
if managed_stream.written_bytes:
written_bytes = managed_stream.written_bytes
else:
written_bytes = os.stat(managed_stream.full_path).st_size
else:
full_path = None
file_name = None
download_directory = None
written_bytes = None
if managed_stream.stream_claim_info:
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
timestamp = self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
confirmations = None if not tx_height else ((best_height + 1) - tx_height if tx_height > 0 else tx_height)
else:
tx_height = None
timestamp = None
confirmations = None
return {
'streaming_url': f"http://{managed_stream.config.streaming_host}:{managed_stream.config.streaming_port}"
f"/stream/{managed_stream.sd_hash}",
'completed': (managed_stream.output_file_exists and (managed_stream.status in ('stopped', 'finished'))
or not managed_stream.saving.is_set()) or all(
managed_stream.blob_manager.is_blob_verified(b.blob_hash)
for b in managed_stream.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
'stopped': not managed_stream.running,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.descriptor.stream_name,
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'download_path': full_path,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
'written_bytes': written_bytes,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
'blobs_remaining': managed_stream.blobs_remaining,
'status': managed_stream.status,
'claim_id': managed_stream.claim_id,
'txid': managed_stream.txid,
'nout': managed_stream.nout,
'outpoint': managed_stream.outpoint,
'claim': managed_stream.claim,
'metadata': managed_stream.metadata,
'protobuf': managed_stream.metadata_protobuf,
'channel_claim_id': managed_stream.channel_claim_id,
'channel_name': managed_stream.channel_name,
'claim_name': managed_stream.claim_name,
'content_fee': managed_stream.content_fee,
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
})
return file
'confirmations': confirmations,
'timestamp': timestamp
}
def encode_claim(self, claim):
encoded = getattr(claim, claim.claim_type).to_dict()
+4 -49
View File
@@ -166,6 +166,10 @@ class ManagedStream:
def claim_name(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def claim(self) -> typing.Optional['Claim']:
return None if not self.stream_claim_info else self.stream_claim_info.claim
@property
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@@ -201,55 +205,6 @@ class ManagedStream:
def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
def as_dict(self) -> typing.Dict:
full_path = self.full_path
file_name = self.file_name
download_directory = self.download_directory
if self.full_path and self.output_file_exists:
if self.written_bytes:
written_bytes = self.written_bytes
else:
written_bytes = os.stat(self.full_path).st_size
else:
full_path = None
file_name = None
download_directory = None
written_bytes = None
return {
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
or not self.saving.is_set()) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
'points_paid': 0.0,
'stopped': not self.running,
'stream_hash': self.stream_hash,
'stream_name': self.descriptor.stream_name,
'suggested_file_name': self.descriptor.suggested_file_name,
'sd_hash': self.descriptor.sd_hash,
'download_path': full_path,
'mime_type': self.mime_type,
'key': self.descriptor.key,
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
'written_bytes': written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
'status': self.status,
'claim_id': self.claim_id,
'txid': self.txid,
'nout': self.nout,
'outpoint': self.outpoint,
'metadata': self.metadata,
'protobuf': self.metadata_protobuf,
'channel_claim_id': self.channel_claim_id,
'channel_name': self.channel_name,
'claim_name': self.claim_name,
'content_fee': self.content_fee
}
@classmethod
async def create(cls, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
file_path: str, key: typing.Optional[bytes] = None,
+15 -3
View File
@@ -10,13 +10,15 @@ if typing.TYPE_CHECKING:
REFLECTOR_V1 = 0
REFLECTOR_V2 = 1
MAX_RESPONSE_SIZE = 2000000
log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.transport: typing.Optional[asyncio.WriteTransport] = None
self.blob_manager = blob_manager
self.descriptor = descriptor
self.response_buff = b''
@@ -37,11 +39,21 @@ class StreamReflectorClient(asyncio.Protocol):
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
def data_received(self, data):
if len(self.response_buff + (data or b'')) > MAX_RESPONSE_SIZE:
log.warning("response message to large from reflector server: %i bytes",
len(self.response_buff + (data or b'')))
self.response_buff = b''
self.transport.close()
return
self.response_buff += (data or b'')
try:
response = json.loads(data.decode())
response = json.loads(self.response_buff.decode())
self.response_buff = b''
self.response_queue.put_nowait(response)
except ValueError:
self.transport.close()
if not data:
log.warning("got undecodable response from reflector server")
self.response_buff = b''
return
async def send_request(self, request_dict: typing.Dict):
+15 -5
View File
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager'):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -28,6 +28,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.sd_blob: typing.Optional['BlobFile'] = None
self.received = []
self.incoming = asyncio.Event(loop=self.loop)
self.chunk_size = response_chunk_size
def connection_made(self, transport):
self.transport = transport
@@ -47,7 +48,15 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.loop.create_task(self.handle_request(request))
def send_response(self, response: typing.Dict):
self.transport.write(json.dumps(response).encode())
def chunk_response(remaining: bytes):
f = self.loop.create_future()
f.add_done_callback(lambda _: self.transport.write(remaining[:self.chunk_size]))
if len(remaining) > self.chunk_size:
f.add_done_callback(lambda _: self.loop.call_soon(chunk_response, remaining[self.chunk_size:]))
self.loop.call_soon(f.set_result, None)
response_bytes = json.dumps(response).encode()
chunk_response(response_bytes)
async def handle_request(self, request: typing.Dict):
if self.client_version is None:
@@ -121,11 +130,12 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer:
def __init__(self, blob_manager: 'BlobManager'):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.response_chunk_size = response_chunk_size
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -133,7 +143,7 @@ class ReflectorServer:
async def _start_server():
server = await self.loop.create_server(
lambda: ReflectorServerProtocol(self.blob_manager),
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
interface, port
)
self.started_listening.set()
+8 -2
View File
@@ -41,8 +41,8 @@ class TestStreamAssembler(AsyncioTestCase):
self.stream = await self.stream_manager.create_stream(file_path)
async def test_reflect_stream(self):
reflector = ReflectorServer(self.server_blob_manager)
async def _test_reflect_stream(self, response_chunk_size):
reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
@@ -63,6 +63,12 @@ class TestStreamAssembler(AsyncioTestCase):
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertListEqual(sent, [])
async def test_reflect_stream(self):
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=50), 3, loop=self.loop)
async def test_reflect_stream_small_response_chunks(self):
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=30), 3, loop=self.loop)
async def test_announces(self):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")