Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6deeb67585 | ||
|
|
c53e76011f | ||
|
|
87c30c5b05 | ||
|
|
359e36a5b8 | ||
|
|
001819d5c2 | ||
|
|
8b4c046d28 | ||
|
|
2c20ad6c43 | ||
|
|
9e610cc54c | ||
|
|
b9d25c6d01 | ||
|
|
419b5b45f2 | ||
|
|
516c2dd5d0 | ||
|
|
b99102f9c9 | ||
|
|
8c6c7b655c | ||
|
|
48c6873fc4 | ||
|
|
15dc52bd9a | ||
|
|
52d555078f | ||
|
|
cc976bd010 |
No files matched your search
@@ -10,7 +10,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.7'
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.7'
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
@@ -138,7 +138,7 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-20.04
|
||||
- ubuntu-18.04
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.7'
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install pyinstaller==5.3
|
||||
- run: pip install pyinstaller==4.6
|
||||
- run: pip install -e .
|
||||
- if: startsWith(github.ref, 'refs/tags/v')
|
||||
run: python docker/set_build.py
|
||||
|
||||
@@ -23,4 +23,4 @@ idea:
|
||||
cp -r scripts/idea/* .idea
|
||||
|
||||
elastic-docker:
|
||||
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
docker run -d --env network.publish_host=127.0.0.1 -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.111.0"
|
||||
__version__ = "0.112.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -42,6 +42,8 @@ class BlobAnnouncer:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
self.announcements_sent_metric.labels(peers=0, error=True).inc()
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
|
||||
@@ -560,6 +560,8 @@ class UPnPComponent(Component):
|
||||
self.upnp = await UPnP.discover(loop=self.component_manager.loop)
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
self.upnp = None
|
||||
|
||||
|
||||
@@ -2943,19 +2943,21 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_channel_sign(
|
||||
self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
|
||||
self, channel_name=None, channel_id=None, hexdata=None, salt=None,
|
||||
channel_account_id=None, wallet_id=None):
|
||||
"""
|
||||
Signs data using the specified channel signing key.
|
||||
|
||||
Usage:
|
||||
channel_sign [<channel_name> | --channel_name=<channel_name>]
|
||||
[<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
|
||||
channel_sign [<channel_name> | --channel_name=<channel_name>] [<channel_id> | --channel_id=<channel_id>]
|
||||
[<hexdata> | --hexdata=<hexdata>] [<salt> | --salt=<salt>]
|
||||
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
|
||||
|
||||
Options:
|
||||
--channel_name=<channel_name> : (str) name of channel used to sign (or use channel id)
|
||||
--channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name)
|
||||
--hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal
|
||||
--salt=<salt> : (str) salt to use for signing, default is to use timestamp
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
@@ -2972,11 +2974,13 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
signing_channel = await self.get_channel_or_error(
|
||||
wallet, channel_account_id, channel_id, channel_name, for_signing=True
|
||||
)
|
||||
timestamp = str(int(time.time()))
|
||||
signature = signing_channel.sign_data(unhexlify(str(hexdata)), timestamp)
|
||||
if salt is None:
|
||||
salt = str(int(time.time()))
|
||||
signature = signing_channel.sign_data(unhexlify(str(hexdata)), salt)
|
||||
return {
|
||||
'signature': signature,
|
||||
'signing_ts': timestamp
|
||||
'signing_ts': salt, # DEPRECATED
|
||||
'salt': salt,
|
||||
}
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
|
||||
@@ -80,6 +80,8 @@ class MarketFeed:
|
||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||
self.last_check = time.time()
|
||||
return self.rate
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||
except json.JSONDecodeError as e:
|
||||
|
||||
@@ -13,11 +13,12 @@ from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Output
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -98,6 +99,8 @@ class FileManager:
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
if 'error' in resolved_result:
|
||||
@@ -192,21 +195,24 @@ class FileManager:
|
||||
####################
|
||||
# make downloader and wait for start
|
||||
####################
|
||||
# temporary with fields we know so downloader can start. Missing fields are populated later.
|
||||
stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name,
|
||||
amount=txo.amount, height=txo.tx_ref.height,
|
||||
serialized=claim.to_bytes().hex())
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
# fixme: this shouldnt be here
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
analytics_manager=self.analytics_manager, claim=stored_claim
|
||||
)
|
||||
stream.downloader.node = source_manager.node
|
||||
else:
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||
status=ManagedStream.STATUS_RUNNING,
|
||||
analytics_manager=self.analytics_manager,
|
||||
status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager,
|
||||
torrent_session=source_manager.torrent_session
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
@@ -243,7 +249,7 @@ class FileManager:
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except (Exception, asyncio.CancelledError) as err: # forgive data timeout, don't delete stream
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
|
||||
if isinstance(err, expected):
|
||||
|
||||
@@ -22,6 +22,8 @@ class BackgroundDownloader:
|
||||
await downloader.download_stream_blob(blob_info)
|
||||
except ValueError:
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.error("Unexpected download error on background downloader")
|
||||
finally:
|
||||
|
||||
@@ -279,7 +279,7 @@ class ManagedStream(ManagedDownloadSource):
|
||||
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6], self.full_path)
|
||||
await self.blob_manager.storage.set_saved_file(self.stream_hash)
|
||||
except (Exception, asyncio.CancelledError) as err:
|
||||
except Exception as err:
|
||||
if os.path.isfile(output_path):
|
||||
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
|
||||
os.remove(output_path)
|
||||
@@ -366,7 +366,7 @@ class ManagedStream(ManagedDownloadSource):
|
||||
return sent
|
||||
except ConnectionError:
|
||||
return sent
|
||||
except (OSError, Exception, asyncio.CancelledError) as err:
|
||||
except (OSError, Exception) as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
|
||||
elif isinstance(err, OSError):
|
||||
|
||||
@@ -164,6 +164,8 @@ class StreamManager(SourceManager):
|
||||
async def reflect_streams(self):
|
||||
try:
|
||||
return await self._reflect_streams()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("reflector task encountered an unexpected error!")
|
||||
|
||||
|
||||
@@ -22,9 +22,9 @@ class TorrentHandle:
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._handle: libtorrent.torrent_handle = handle
|
||||
self.started = asyncio.Event()
|
||||
self.finished = asyncio.Event()
|
||||
self.metadata_completed = asyncio.Event()
|
||||
self.started = asyncio.Event(loop=loop)
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
self.metadata_completed = asyncio.Event(loop=loop)
|
||||
self.size = 0
|
||||
self.total_wanted_done = 0
|
||||
self.name = ''
|
||||
|
||||
@@ -141,7 +141,7 @@ class CoinSelector:
|
||||
_) -> List[OutputEffectiveAmountEstimator]:
|
||||
""" Accumulate UTXOs at random until there is enough to cover the target. """
|
||||
target = self.target + self.cost_of_change
|
||||
self.random.shuffle(txos, random=self.random.random) # pylint: disable=deprecated-argument
|
||||
self.random.shuffle(txos, self.random.random)
|
||||
selection = []
|
||||
amount = 0
|
||||
for coin in txos:
|
||||
|
||||
+18
-7
@@ -30,6 +30,7 @@ from lbry.wallet.bip32 import PublicKey, PrivateKey
|
||||
from lbry.wallet.coinselection import CoinSelector
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel("DEBUG")
|
||||
|
||||
LedgerType = Type['BaseLedger']
|
||||
|
||||
@@ -506,6 +507,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
|
||||
def process_status_update(self, update):
|
||||
address, remote_status = update
|
||||
print(f"**** status update {address} {remote_status}")
|
||||
self._update_tasks.add(self.update_history(address, remote_status))
|
||||
|
||||
async def update_history(self, address, remote_status, address_manager: AddressManager = None,
|
||||
@@ -518,6 +520,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return True
|
||||
|
||||
remote_history = await self.network.retriable_call(self.network.get_history, address)
|
||||
print(f'>>>>>> {remote_history}')
|
||||
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
|
||||
we_need = set(remote_history) - set(local_history)
|
||||
if not we_need:
|
||||
@@ -549,7 +552,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
continue
|
||||
to_request[i] = (txid, remote_height)
|
||||
|
||||
log.debug(
|
||||
log.warning(
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
|
||||
len(remote_history), address
|
||||
)
|
||||
@@ -558,8 +561,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self.maybe_has_channel_key(tx)
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
if len(pending_synced_history) % 100 == 0:
|
||||
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
log.warning("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
log.warning("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
|
||||
assert len(pending_synced_history) == len(remote_history), \
|
||||
f"{len(pending_synced_history)} vs {len(remote_history)} for {address}"
|
||||
@@ -606,7 +609,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._known_addresses_out_of_sync.add(address)
|
||||
return False
|
||||
else:
|
||||
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
|
||||
log.warning("finished syncing transaction history for %s, %i known txs", address, len(local_history))
|
||||
return True
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
|
||||
@@ -621,6 +624,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
header = await self.headers.get(remote_height)
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
if not tx.is_verified:
|
||||
print(f"&&&&&&& {tx.height}: {merkle_root} != {header['merkle_root']}")
|
||||
return tx
|
||||
|
||||
def maybe_has_channel_key(self, tx):
|
||||
@@ -938,7 +943,9 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
|
||||
account.id, balance, total_receiving, account.receiving.gap, total_change,
|
||||
account.change.gap, channel_count, len(account.channel_keys), claim_count)
|
||||
except Exception:
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception(
|
||||
'Failed to display wallet state, please file issue '
|
||||
'for this bug along with the traceback you see below:')
|
||||
@@ -961,7 +968,9 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
claim_ids = [p.purchased_claim_id for p in purchases]
|
||||
try:
|
||||
resolved, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
|
||||
except Exception:
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception("Resolve failed while looking up purchased claim ids:")
|
||||
resolved = []
|
||||
lookup = {claim.claim_id: claim for claim in resolved}
|
||||
@@ -1041,7 +1050,9 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
|
||||
try:
|
||||
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
|
||||
except Exception:
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception("Resolve failed while looking up collection claim ids:")
|
||||
return []
|
||||
claims = []
|
||||
|
||||
@@ -76,7 +76,9 @@ class ClientSession(BaseClientSession):
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
try:
|
||||
return request.result()
|
||||
result = request.result()
|
||||
log.warning("sent %s%s to %s:%i (%i timeout) result: %s", method, tuple(args), self.server[0], self.server[1], self.timeout, result)
|
||||
return result
|
||||
except ConnectionResetError:
|
||||
log.error(
|
||||
"wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes",
|
||||
@@ -117,7 +119,7 @@ class ClientSession(BaseClientSession):
|
||||
)
|
||||
else:
|
||||
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
|
||||
except (Exception, asyncio.CancelledError) as err:
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.info("closing connection to %s:%i", *self.server)
|
||||
else:
|
||||
@@ -304,7 +306,7 @@ class Network:
|
||||
await client.ensure_server_version()
|
||||
return client
|
||||
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
|
||||
log.warning("Connecting to %s:%d failed", host, port)
|
||||
log.exception("Connecting to %s:%d failed", host, port)
|
||||
client._close()
|
||||
return
|
||||
|
||||
@@ -328,7 +330,8 @@ class Network:
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.debug("discover other hubs %s:%i", *client.server)
|
||||
await self._update_hubs(await client.send_request('server.peers.subscribe', []))
|
||||
# TODO: Enable this after herald.go supports 'server.peers.subscribe'.
|
||||
#await self._update_hubs(await client.send_request('server.peers.subscribe', []))
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# pylint: disable=import-error
|
||||
import os
|
||||
import signal
|
||||
import json
|
||||
import shutil
|
||||
import asyncio
|
||||
@@ -30,7 +31,7 @@ try:
|
||||
from hub.elastic_sync.service import ElasticSyncService
|
||||
from hub.scribe.service import BlockchainProcessorService
|
||||
except ImportError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def get_lbcd_node_from_ledger(ledger_module):
|
||||
@@ -214,6 +215,7 @@ class SPVNode:
|
||||
self.port = 50001 + node_number # avoid conflict with default daemon
|
||||
self.udp_port = self.port
|
||||
self.elastic_notifier_port = 19080 + node_number
|
||||
self.elastic_services = f'localhost:9200/localhost:{self.elastic_notifier_port}'
|
||||
self.session_timeout = 600
|
||||
self.stopped = True
|
||||
self.index_name = uuid4().hex
|
||||
@@ -225,6 +227,7 @@ class SPVNode:
|
||||
self.stopped = False
|
||||
try:
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
#self.data_path = '/Users/swdev1/herald/test_db'
|
||||
conf = {
|
||||
'description': '',
|
||||
'payment_address': '',
|
||||
@@ -235,7 +238,7 @@ class SPVNode:
|
||||
'host': self.hostname,
|
||||
'tcp_port': self.port,
|
||||
'udp_port': self.udp_port,
|
||||
'elastic_notifier_port': self.elastic_notifier_port,
|
||||
'elastic_services': self.elastic_services,
|
||||
'session_timeout': self.session_timeout,
|
||||
'max_query_workers': 0,
|
||||
'es_index_prefix': self.index_name,
|
||||
@@ -248,7 +251,9 @@ class SPVNode:
|
||||
BlockchainEnv(db_dir=self.data_path, daemon_url=lbcwallet_node.rpc_url,
|
||||
reorg_limit=100, max_query_workers=0, chain='regtest', index_address_status=False)
|
||||
)
|
||||
self.server = HubServerService(ServerEnv(**conf))
|
||||
# Select Herald variant:
|
||||
self.server = HubNode("", "herald", self) # Go Herald
|
||||
#self.server = HubServerService(ServerEnv(**conf)) # Python Herald
|
||||
self.es_writer = ElasticSyncService(
|
||||
ElasticEnv(
|
||||
db_dir=self.data_path, reorg_limit=100, max_query_workers=0, chain='regtest',
|
||||
@@ -284,7 +289,8 @@ class SPVNode:
|
||||
cleanup and self.cleanup()
|
||||
|
||||
def cleanup(self):
|
||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
log.error("skipping cleanup of data_path: %s", self.data_path)
|
||||
#shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
|
||||
|
||||
class LBCDProcess(asyncio.SubprocessProtocol):
|
||||
@@ -673,3 +679,146 @@ class LBCWalletNode:
|
||||
|
||||
def get_raw_transaction(self, txid):
|
||||
return self._cli_cmnd('getrawtransaction', txid, '1')
|
||||
|
||||
|
||||
class HubProcess(asyncio.SubprocessProtocol):
|
||||
def __init__(self, ready, stopped):
|
||||
self.ready = ready
|
||||
self.stopped = stopped
|
||||
self.log = log.getChild('hub')
|
||||
self.transport = None
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
self.stopped.clear()
|
||||
self.ready.set()
|
||||
if self.log:
|
||||
self.log.warning(data.decode())
|
||||
#if b'error' in data.lower():
|
||||
# self.ready.set()
|
||||
# raise SystemError(data.decode())
|
||||
if b'listening on' in data:
|
||||
self.ready.set()
|
||||
str_lines = str(data.decode()).split("\n")
|
||||
for line in str_lines:
|
||||
if 'releaseTime' in line:
|
||||
print(line)
|
||||
|
||||
def process_exited(self):
|
||||
self.ready.clear()
|
||||
self.stopped.set()
|
||||
|
||||
async def stop(self):
|
||||
t = asyncio.create_task(self.stopped.wait())
|
||||
try:
|
||||
self.transport.send_signal(signal.SIGINT)
|
||||
await asyncio.wait_for(t, 3)
|
||||
# log.warning("stopped go hub")
|
||||
except asyncio.TimeoutError:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
self.transport.terminate()
|
||||
await self.stopped.wait()
|
||||
log.warning("terminated go hub")
|
||||
|
||||
|
||||
class HubNode:
|
||||
def __init__(self, url, daemon, spv_node):
|
||||
self.spv_node = spv_node
|
||||
self.latest_release_url = url
|
||||
self.project_dir = os.path.dirname(os.path.dirname(__file__))
|
||||
self.bin_dir = os.path.join(self.project_dir, 'bin')
|
||||
self.daemon_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.cli_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.log = log.getChild('hub')
|
||||
self.transport = None
|
||||
self.protocol = None
|
||||
self.hostname = 'localhost'
|
||||
self.rpcport = 50051 # avoid conflict with default rpc port
|
||||
self._stopped = asyncio.Event()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
def stopped(self):
|
||||
return not self.running.is_set()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
os.path.exists(self.cli_bin) and
|
||||
os.path.exists(self.daemon_bin)
|
||||
)
|
||||
|
||||
def download(self):
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir,
|
||||
self.latest_release_url[self.latest_release_url.rfind('/')+1:]
|
||||
)
|
||||
|
||||
if not os.path.exists(self.bin_dir):
|
||||
os.mkdir(self.bin_dir)
|
||||
|
||||
if not os.path.exists(downloaded_file):
|
||||
self.log.info('Downloading: %s', self.latest_release_url)
|
||||
with urllib.request.urlopen(self.latest_release_url) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
shutil.copyfileobj(response, out_file)
|
||||
|
||||
self.log.info('Extracting: %s', downloaded_file)
|
||||
|
||||
if downloaded_file.endswith('.zip'):
|
||||
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||
dotzip.extractall(self.bin_dir)
|
||||
# zipfile bug https://bugs.python.org/issue15795
|
||||
os.chmod(self.cli_bin, 0o755)
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
elif downloaded_file.endswith('.tar.gz'):
|
||||
with tarfile.open(downloaded_file) as tar:
|
||||
tar.extractall(self.bin_dir)
|
||||
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
return self.exists
|
||||
|
||||
def ensure(self):
|
||||
return self.exists or self.download()
|
||||
|
||||
async def start(self):
|
||||
assert self.ensure()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.get_child_watcher().attach_loop(loop)
|
||||
command = [
|
||||
self.daemon_bin, 'serve',
|
||||
'--db-path', self.spv_node.data_path + '/lbry-rocksdb',
|
||||
'--chain', 'regtest',
|
||||
'--json-rpc-port', str(self.spv_node.port),
|
||||
'--json-rpc-http-port', '0', # disabled
|
||||
'--esindex', self.spv_node.index_name + 'claims',
|
||||
'--notifier-port', str(self.spv_node.elastic_notifier_port),
|
||||
'--debug'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
self.protocol = HubProcess(self.running, self._stopped)
|
||||
try:
|
||||
self.transport, _ = await loop.subprocess_exec(
|
||||
lambda: self.protocol, *command
|
||||
)
|
||||
self.protocol.transport = self.transport
|
||||
except Exception as e:
|
||||
log.exception('failed to start go hub', exc_info=e)
|
||||
raise e
|
||||
await self.protocol.ready.wait()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
try:
|
||||
if self.protocol:
|
||||
await self.protocol.stop()
|
||||
except Exception as e:
|
||||
log.exception('failed to stop go hub', exc_info=e)
|
||||
raise e
|
||||
finally:
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
@@ -182,6 +182,8 @@ class Wallet:
|
||||
raise InvalidPasswordError()
|
||||
if "unknown compression method" in e.args[0].lower():
|
||||
raise InvalidPasswordError()
|
||||
if "invalid window size" in e.args[0].lower():
|
||||
raise InvalidPasswordError()
|
||||
raise
|
||||
return json.loads(decompressed)
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ setup(
|
||||
long_description_content_type="text/markdown",
|
||||
keywords="lbry protocol media",
|
||||
license='MIT',
|
||||
python_requires='>=3.9',
|
||||
python_requires='>=3.7',
|
||||
packages=find_packages(exclude=('tests',)),
|
||||
zip_safe=False,
|
||||
entry_points={
|
||||
@@ -36,11 +36,11 @@ setup(
|
||||
'distro==1.4.0',
|
||||
'base58==1.0.0',
|
||||
'cffi==1.13.2',
|
||||
'cryptography==3.4.7',
|
||||
'cryptography==2.5',
|
||||
'protobuf==3.17.2',
|
||||
'prometheus_client==0.7.1',
|
||||
'ecdsa==0.13.3',
|
||||
'pyyaml==5.4',
|
||||
'pyyaml==5.3.1',
|
||||
'docopt==0.6.2',
|
||||
'hachoir==3.1.2',
|
||||
'coincurve==15.0.0',
|
||||
@@ -57,7 +57,7 @@ setup(
|
||||
'jsonschema==4.4.0',
|
||||
],
|
||||
'hub': [
|
||||
'hub@git+https://github.com/lbryio/hub.git@024aceda53fe6d1ab8d519b73584437c25de6975'
|
||||
'hub@git+https://github.com/lbryio/hub.git@929448d64bcbe6c5e476757ec78456beaa85e56a'
|
||||
]
|
||||
},
|
||||
classifiers=[
|
||||
|
||||
@@ -31,7 +31,7 @@ STREAM_TYPES = {
|
||||
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
signature['signing_ts'].encode(),
|
||||
signature['salt'].encode(),
|
||||
channel_hash or channel.claim_hash,
|
||||
data
|
||||
]
|
||||
@@ -1239,8 +1239,13 @@ class ChannelCommands(CommandTestCase):
|
||||
channel = channel_tx.outputs[0]
|
||||
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
|
||||
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
|
||||
signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef'))
|
||||
signature4 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign, salt='beef'))
|
||||
self.assertNotEqual(signature2, signature3)
|
||||
self.assertEqual(signature3, signature4)
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature3))
|
||||
signature3 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=99))
|
||||
self.assertTrue(verify(channel, unhexlify('99'), signature3))
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import unittest
|
||||
from unittest import skipIf, skip
|
||||
from unittest import skipIf
|
||||
import asyncio
|
||||
import os
|
||||
from binascii import hexlify
|
||||
@@ -51,8 +51,7 @@ class FileCommands(CommandTestCase):
|
||||
self.addCleanup(task.cancel)
|
||||
return tx, btih
|
||||
|
||||
#@skipIf(TorrentSession is None, "libtorrent not installed")
|
||||
@skip
|
||||
@skipIf(TorrentSession is None, "libtorrent not installed")
|
||||
async def test_download_torrent(self):
|
||||
tx, btih = await self.initialize_torrent()
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
@@ -90,6 +89,21 @@ class FileCommands(CommandTestCase):
|
||||
await self.reflector.blob_manager.delete_blobs(all_except_sd)
|
||||
self.assertEqual(all_except_sd, await self.daemon.jsonrpc_file_reflect(sd_hash=sd_hash))
|
||||
|
||||
async def test_sd_blob_fields_fallback(self):
|
||||
claim_id = self.get_claim_id(await self.stream_create('foo', '0.01', suffix='.txt'))
|
||||
stream = (await self.daemon.jsonrpc_file_list())["items"][0]
|
||||
stream.descriptor.suggested_file_name = ' '
|
||||
stream.descriptor.stream_name = ' '
|
||||
stream.descriptor.stream_hash = stream.descriptor.get_stream_hash()
|
||||
sd_hash = stream.descriptor.sd_hash = stream.descriptor.calculate_sd_hash()
|
||||
await stream.descriptor.make_sd_blob()
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
await self.stream_update(claim_id=claim_id, sd_hash=sd_hash)
|
||||
file_dict = await self.out(self.daemon.jsonrpc_get('lbry://foo', save_file=True))
|
||||
self.assertEqual(file_dict['suggested_file_name'], stream.file_name)
|
||||
self.assertEqual(file_dict['stream_name'], stream.file_name)
|
||||
self.assertEqual(file_dict['mime_type'], 'text/plain')
|
||||
|
||||
async def test_file_management(self):
|
||||
await self.stream_create('foo', '0.01')
|
||||
await self.stream_create('foo2', '0.01')
|
||||
|
||||
@@ -1508,27 +1508,27 @@ class ResolveClaimTakeovers(BaseResolveTestCase):
|
||||
COIN = int(1E8)
|
||||
|
||||
self.assertEqual(self.conductor.spv_node.writer.height, 207)
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
|
||||
(208, bytes.fromhex(claim_id1)), (0, 10 * COIN)
|
||||
)
|
||||
await self.generate(1)
|
||||
self.assertEqual(self.conductor.spv_node.writer.height, 208)
|
||||
|
||||
self.assertEqual(1.7090807854206793, await get_trending_score(claim_id1))
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
|
||||
(209, bytes.fromhex(claim_id1)), (10 * COIN, 100 * COIN)
|
||||
)
|
||||
await self.generate(1)
|
||||
self.assertEqual(self.conductor.spv_node.writer.height, 209)
|
||||
self.assertEqual(2.2437974397778886, await get_trending_score(claim_id1))
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
|
||||
(309, bytes.fromhex(claim_id1)), (100 * COIN, 1000000 * COIN)
|
||||
)
|
||||
await self.generate(100)
|
||||
self.assertEqual(self.conductor.spv_node.writer.height, 309)
|
||||
self.assertEqual(5.157053472135866, await get_trending_score(claim_id1))
|
||||
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stage_put(
|
||||
self.conductor.spv_node.writer.db.prefix_db.trending_notification.stash_put(
|
||||
(409, bytes.fromhex(claim_id1)), (1000000 * COIN, 1 * COIN)
|
||||
)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import asyncio
|
||||
import unittest
|
||||
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
from lbry.wallet import Transaction
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
@@ -29,17 +29,42 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
# someone's tx
|
||||
change_address = await self.blockchain.get_raw_change_address()
|
||||
sendtxid = await self.blockchain.send_to_address(change_address, 10)
|
||||
await asyncio.sleep(0.2)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.id, sendtxid)
|
||||
self.assertEqual(tx.height, -1)
|
||||
# After a few tries, Hub should have the transaction (in mempool).
|
||||
for i in range(5):
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
# Retry if Hub is not aware of the transaction.
|
||||
if isinstance(tx, dict):
|
||||
# Fields: 'success', 'code', 'message'
|
||||
self.assertFalse(tx['success'], tx)
|
||||
self.assertEqual(tx['code'], 404, tx)
|
||||
self.assertEqual(tx['message'], "transaction not found", tx)
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
break
|
||||
# verify transaction show (in mempool)
|
||||
self.assertTrue(isinstance(tx, Transaction), str(tx))
|
||||
# Fields: 'txid', 'raw', 'height', 'position', 'is_verified', and more.
|
||||
self.assertEqual(tx.id, sendtxid, vars(tx))
|
||||
self.assertEqual(tx.height, -1, vars(tx))
|
||||
self.assertEqual(tx.is_verified, False, vars(tx))
|
||||
|
||||
# transaction is confirmed and leaves mempool
|
||||
await self.generate(1)
|
||||
|
||||
# verify transaction show
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.height, self.ledger.headers.height)
|
||||
self.assertTrue(isinstance(tx, Transaction), str(tx))
|
||||
self.assertEqual(tx.id, sendtxid, vars(tx))
|
||||
self.assertEqual(tx.height, self.ledger.headers.height, vars(tx))
|
||||
self.assertEqual(tx.is_verified, True, vars(tx))
|
||||
|
||||
# inexistent
|
||||
result = await self.daemon.jsonrpc_transaction_show('0'*64)
|
||||
self.assertFalse(result['success'])
|
||||
self.assertTrue(isinstance(result, dict), result)
|
||||
# Fields: 'success', 'code', 'message'
|
||||
self.assertFalse(result['success'], result)
|
||||
self.assertEqual(result['code'], 404, result)
|
||||
self.assertEqual(result['message'], "transaction not found", result)
|
||||
|
||||
async def test_utxo_release(self):
|
||||
await self.send_to_address_and_wait(
|
||||
|
||||
@@ -305,7 +305,6 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.assertEqual(stored_status, "running")
|
||||
|
||||
await stream.stop()
|
||||
await asyncio.sleep(1) # TODO: should not be needed
|
||||
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
@@ -341,6 +340,8 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
try:
|
||||
await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
error = err
|
||||
self.assertEqual(expected_error, type(error))
|
||||
|
||||
|
||||
@@ -470,7 +470,7 @@ class TestUpgrade(AsyncioTestCase):
|
||||
|
||||
|
||||
class TestSQLiteRace(AsyncioTestCase):
|
||||
max_misuse_attempts = 80000
|
||||
max_misuse_attempts = 120000
|
||||
|
||||
def setup_db(self):
|
||||
self.db = sqlite3.connect(":memory:", isolation_level=None)
|
||||
|
||||
Reference in new issue
Block a user