Compare commits

...
Author SHA1 Message Date
Jack Robison db8fb15cc5 0.37.2rc4 2019-05-24 18:12:19 -04:00
Jack Robison a165ad910f improve set_build.py, pin torba version
- adds BUILD_COMMIT to lbrynet.build_type
2019-05-24 18:11:50 -04:00
Jack Robison b711069456 fix false negative with file completed field 2019-05-24 17:31:26 -04:00
Jack Robison c977a672b0 start returning range request bytes at the requested offset 2019-05-24 17:31:26 -04:00
Jack Robison ac5794d96d add blob_lru_cache_size config setting, defaults to 32 2019-05-24 17:31:26 -04:00
15 changed files with 157 additions and 51 deletions

No files matched your search

+10 -6
View File
@@ -3,6 +3,9 @@ dist: xenial
language: python
python: "3.7"
env:
- TORBA=v0.5.3
jobs:
include:
@@ -10,7 +13,7 @@ jobs:
name: "pylint lbrynet"
install:
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script: pylint lbrynet
@@ -18,7 +21,7 @@ jobs:
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
@@ -29,7 +32,8 @@ jobs:
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
@@ -38,7 +42,7 @@ jobs:
- name: "Run Examples"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
@@ -55,7 +59,7 @@ jobs:
- docker pull lbry/pyinstaller34_32bits:py371
script:
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh ${TORBA}
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
@@ -79,7 +83,7 @@ jobs:
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- python3 scripts/set_build.py
- pip3 install -e .
script:
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.37.1"
__version__ = "0.37.2rc4"
version = tuple(__version__.split('.'))
+3
View File
@@ -2,6 +2,7 @@ import os
import typing
import asyncio
import logging
from lbrynet.utils import LRUCache
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
@@ -30,6 +31,8 @@ class BlobManager:
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.config.blob_lru_cache_size)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
+1
View File
@@ -1,2 +1,3 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
+4 -1
View File
@@ -485,7 +485,10 @@ class Config(CLIConfig):
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32
)
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
+15
View File
@@ -485,6 +485,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def handle_stream_get_request(self, request: web.Request):
if not self.conf.streaming_get:
log.warning("streaming_get is disabled, rejecting request")
raise web.HTTPForbidden()
name_and_claim_id = request.path.split("/get/")[1]
if "/" not in name_and_claim_id:
@@ -500,6 +501,20 @@ class Daemon(metaclass=JSONRPCServerType):
raise web.HTTPFound(f"/stream/{stream.sd_hash}")
async def handle_stream_range_request(self, request: web.Request):
try:
return await self._handle_stream_range_request(request)
except web.HTTPException as err:
log.warning("http code during /stream range request: %s", err)
raise err
except asyncio.CancelledError:
log.debug("/stream range request cancelled")
except Exception:
log.exception("error handling /stream range request")
raise
finally:
log.debug("finished handling /stream range request")
async def _handle_stream_range_request(self, request: web.Request):
sd_hash = request.path.split("/stream/")[1]
if not self.stream_manager.started.is_set():
await self.stream_manager.started.wait()
+11 -1
View File
@@ -3,7 +3,7 @@ import typing
import logging
import binascii
from lbrynet.error import DownloadSDTimeout
from lbrynet.utils import resolve_host
from lbrynet.utils import resolve_host, lru_cache_concurrent
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
@@ -36,6 +36,16 @@ class StreamDownloader:
self.time_to_descriptor: typing.Optional[float] = None
self.time_to_first_bytes: typing.Optional[float] = None
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
return await self.read_blob(blob_info, 2)
if self.blob_manager.decrypted_blob_lru_cache:
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
cached_read_blob
)
self.cached_read_blob = cached_read_blob
async def add_fixed_peers(self):
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
+31 -10
View File
@@ -3,7 +3,7 @@ import asyncio
import typing
import logging
import binascii
from aiohttp.web import Request, StreamResponse
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbrynet.utils import generate_id
from lbrynet.error import DownloadSDTimeout
from lbrynet.schema.mime_types import guess_media_type
@@ -42,6 +42,9 @@ class ManagedStream:
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
SAVING_ID = 1
STREAMING_ID = 2
__slots__ = [
'loop',
'config',
@@ -214,7 +217,8 @@ class ManagedStream:
written_bytes = None
return {
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': (self.output_file_exists and self.status in ('stopped', 'finished')) or all(
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
or not self.saving.is_set()) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
@@ -304,14 +308,19 @@ class ManagedStream:
raise IndexError(start_blob_num)
for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
assert i + start_blob_num == blob_info.blob_num
decrypted = await self.downloader.read_blob(blob_info, connection_id)
if connection_id == self.STREAMING_ID:
decrypted = await self.downloader.cached_read_blob(blob_info)
else:
decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted)
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-')
)
await self.start(node)
headers, size, skip_blobs = self._prepare_range_response_headers(request.headers.get('range', 'bytes=0-'))
response = StreamResponse(
status=206,
headers=headers
@@ -321,11 +330,16 @@ class ManagedStream:
self.streaming.set()
try:
wrote = 0
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=2):
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID):
if not wrote:
decrypted = decrypted[first_blob_start_offset:]
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1,
len(self.descriptor.blobs) - 1)
await response.write_eof(decrypted)
else:
log.debug("sending browser blob (%i/%i)", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await response.write(decrypted)
wrote += len(decrypted)
log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "",
@@ -354,7 +368,7 @@ class ManagedStream:
self.started_writing.clear()
try:
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=1):
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
self.written_bytes += len(decrypted)
@@ -485,7 +499,7 @@ class ManagedStream:
return
await asyncio.sleep(1, loop=self.loop)
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int]:
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
if '=' in get_range:
get_range = get_range.split('=')[1]
start, end = get_range.split('-')
@@ -503,16 +517,23 @@ class ManagedStream:
log.debug("estimating stream size")
start = int(start)
if not 0 <= start < size:
raise HTTPRequestRangeNotSatisfiable()
end = int(end) if end else size - 1
if end >= size:
raise HTTPRequestRangeNotSatisfiable()
skip_blobs = start // 2097150
skip = skip_blobs * 2097151
start = skip
skip_first_blob = start - skip
start = skip_first_blob + skip
final_size = end - start + 1
headers = {
'Accept-Ranges': 'bytes',
'Content-Range': f'bytes {start}-{end}/{size}',
'Content-Length': str(final_size),
'Content-Type': self.mime_type
}
return headers, size, skip_blobs
return headers, size, skip_blobs, skip_first_blob
+2
View File
@@ -62,6 +62,7 @@ class CommandTestCase(IntegrationTestCase):
LEDGER = lbrynet.wallet
MANAGER = LbryWalletManager
VERBOSITY = logging.WARN
blob_lru_cache_size = 0
async def asyncSetUp(self):
await super().asyncSetUp()
@@ -81,6 +82,7 @@ class CommandTestCase(IntegrationTestCase):
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
+4 -3
View File
@@ -229,11 +229,12 @@ class LRUCache:
return item in self.cache
def lru_cache_concurrent(cache_size: int):
if not cache_size > 0:
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None):
if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size")
concurrent_cache = {}
lru_cache = LRUCache(cache_size)
lru_cache = override_lru_cache or LRUCache(cache_size)
def wrapper(async_fn):
+26 -24
View File
@@ -1,33 +1,35 @@
"""Set the build version to be 'qa', 'rc', 'release'"""
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
import os.path
import re
import subprocess
import sys
import os
import re
import logging
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
def get_build_type(travis_tag=None):
if not travis_tag:
return "qa"
log.debug("getting build type for tag: \"%s\"", travis_tag)
if re.match('v\d+\.\d+\.\d+rc\d+$', travis_tag):
return 'rc'
elif re.match('v\d+\.\d+\.\d+$', travis_tag):
return 'release'
return 'qa'
def main():
build = get_build()
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
f.write("BUILD = '{}'\n".format(build))
def get_build():
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
print('Build: rc')
return 'rc'
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
print('Build: release')
return 'release'
print('Build: qa')
return 'qa'
except subprocess.CalledProcessError:
print("Couldn't determine build type, defaulting to qa.")
return 'qa'
build_type_path = os.path.join(root_dir, 'lbrynet', 'build_type.py')
log.debug("configuring build type file: %s", build_type_path)
travis_commit = os.environ['TRAVIS_COMMIT'][:6]
build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
with open(build_type_path, 'w') as f:
f.write("BUILD = \"{}\"\nBUILD_COMMIT = \"{}\"\n".format(build_type, travis_commit))
if __name__ == '__main__':
+3 -4
View File
@@ -1,13 +1,10 @@
set -x
TORBA=$1
rm -rf /tmp/.wine-*
apt-get -qq update
apt-get -qq install -y git
pip install setuptools_scm
git clone https://github.com/lbryio/torba.git --depth 1
cd torba && pip install -e . && cd ..
cd lbry
@@ -15,6 +12,8 @@ cd lbry
wget -Onetifaces-0.10.7-cp37-cp37m-win32.whl https://ci.appveyor.com/api/buildjobs/6hworunifsymrhp2/artifacts/dist%2Fnetifaces-0.10.7-cp37-cp37m-win32.whl
pip install netifaces-0.10.7-cp37-cp37m-win32.whl
pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
pip install -e .
pip install pywin32
+1 -1
View File
@@ -23,7 +23,7 @@ setup(
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
},
install_requires=[
'torba',
'torba==0.5.3',
'aiohttp==3.5.4',
'aioupnp',
'appdirs==1.4.3',
+44
View File
@@ -2,6 +2,7 @@ import os
import hashlib
import aiohttp
import aiohttp.web
import asyncio
from lbrynet.utils import aiohttp_request
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
@@ -373,3 +374,46 @@ class RangeRequests(CommandTestCase):
await stream.finished_writing.wait()
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
class RangeRequestsLRUCache(CommandTestCase):
blob_lru_cache_size = 32
async def _request_stream(self):
name = 'foo'
url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}'
async with aiohttp_request('get', url) as req:
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
content_range = req.headers.get('Content-Range')
content_length = int(req.headers.get('Content-Length'))
streamed_bytes = await req.content.read()
self.assertEqual(content_length, len(streamed_bytes))
self.assertEqual(15, content_length)
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed_bytes)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_with_blob_lru_cache(self):
self.data = b'hi'
self.daemon.conf.save_blobs = False
self.daemon.conf.save_files = False
await self.stream_create('foo', '0.01', data=self.data, file_size=0)
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
await self.daemon.streaming_runner.setup()
site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host,
self.daemon.conf.streaming_port)
await site.start()
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
await self._request_stream()
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
self.server.stop_server()
# running with cache size 0 gets through without errors without
# this since the server doesnt stop immediately
await asyncio.sleep(1, loop=self.loop)
await self._request_stream()
+1
View File
@@ -5,6 +5,7 @@ envlist = py37-integration
deps =
coverage
../torba
extras = test
changedir = {toxinidir}/tests
setenv =