Compare commits
432
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf73e4599f | ||
|
|
2a698932da | ||
|
|
ee36162b16 | ||
|
|
97f472d4e1 | ||
|
|
160e227e90 | ||
|
|
475ba010e3 | ||
|
|
57ebbbcb78 | ||
|
|
cbf973bf2d | ||
|
|
9d26ad96c6 | ||
|
|
c2a3ec3265 | ||
|
|
521d783260 | ||
|
|
50846cd37e | ||
|
|
40ed5f311b | ||
|
|
293410ac9f | ||
|
|
068f46d137 | ||
|
|
5c60c09fef | ||
|
|
aca4a243d6 | ||
|
|
422f292d82 | ||
|
|
c81598aab8 | ||
|
|
3f8c8e01c6 | ||
|
|
8d90653395 | ||
|
|
01841694ad | ||
|
|
c3d0dd3073 | ||
|
|
4851f5c300 | ||
|
|
aef5f11a18 | ||
|
|
700f1cb8e5 | ||
|
|
7e5f7c1725 | ||
|
|
aa7a17b112 | ||
|
|
838fd71538 | ||
|
|
0146044b90 | ||
|
|
f8f73ed4a6 | ||
|
|
74bf8c551b | ||
|
|
4be346d0bb | ||
|
|
4fd1fe0046 | ||
|
|
9524f03eac | ||
|
|
90c8a8bea3 | ||
|
|
154c41bd8d | ||
|
|
f271511484 | ||
|
|
dde4a5b495 | ||
|
|
68a940f3a6 | ||
|
|
8c773151a3 | ||
|
|
f1f5276a04 | ||
|
|
51490b36e6 | ||
|
|
e00dc78e1f | ||
|
|
72beb02ec1 | ||
|
|
b13c346418 | ||
|
|
2520465fc1 | ||
|
|
dd868d5148 | ||
|
|
e90fb9a28f | ||
|
|
96d1745232 | ||
|
|
cab5f9188f | ||
|
|
44e647d85d | ||
|
|
5686c2ade9 | ||
|
|
1cc321e9cd | ||
|
|
b4d144da21 | ||
|
|
39c2c0b922 | ||
|
|
dbf11fe750 | ||
|
|
cc391324e4 | ||
|
|
273b4f6d3b | ||
|
|
d359d25935 | ||
|
|
645a81cec2 | ||
|
|
06da53ef09 | ||
|
|
692af0cc55 | ||
|
|
32b95fa04a | ||
|
|
5ebcbc6370 | ||
|
|
3978d64056 | ||
|
|
d35f7a5ed5 | ||
|
|
2f84c38f6a | ||
|
|
b656759e0e | ||
|
|
15ac365463 | ||
|
|
2ab78a92ca | ||
|
|
71b5c4c6ad | ||
|
|
eae1a0912b | ||
|
|
d5ec647f46 | ||
|
|
e066b2a9da | ||
|
|
8adec9d0ad | ||
|
|
030e407584 | ||
|
|
7ed807e41b | ||
|
|
354cc9b22d | ||
|
|
ff51a5bdc6 | ||
|
|
8618be1b5c | ||
|
|
ebe351c67e | ||
|
|
6ad2e31976 | ||
|
|
d44dc103d5 | ||
|
|
331ce1f12c | ||
|
|
37c408a6e7 | ||
|
|
a866621ccb | ||
|
|
b45b87e72d | ||
|
|
300ae6b134 | ||
|
|
0cb66e1f0d | ||
|
|
618f739049 | ||
|
|
5c8329e951 | ||
|
|
683f6b366d | ||
|
|
ce63c2281f | ||
|
|
251cb4925a | ||
|
|
f8fe205066 | ||
|
|
9908775576 | ||
|
|
bf7745ec8d | ||
|
|
1aa296e168 | ||
|
|
171c99b12b | ||
|
|
83bb12265e | ||
|
|
b4e99faba7 | ||
|
|
78b559b274 | ||
|
|
974b933612 | ||
|
|
3aac113d45 | ||
|
|
a04d50a3bc | ||
|
|
6ec30798ca | ||
|
|
b4d47e423d | ||
|
|
bf86e7658e | ||
|
|
98c0200c53 | ||
|
|
a850fb64c8 | ||
|
|
02b0f3e9f2 | ||
|
|
948a58f628 | ||
|
|
21a2076f26 | ||
|
|
efd8fd81ed | ||
|
|
5e73f82c23 | ||
|
|
01691dd92a | ||
|
|
5fe339c53b | ||
|
|
c4b86454b5 | ||
|
|
60969e4817 | ||
|
|
48bb84fc1e | ||
|
|
b055c25156 | ||
|
|
35905b99ff | ||
|
|
806857c066 | ||
|
|
ba8a7fc351 | ||
|
|
a36fd76eb1 | ||
|
|
02994164be | ||
|
|
c3b89a896b | ||
|
|
b0bf6eff16 | ||
|
|
47c4cb800d | ||
|
|
c5f6a4ca1c | ||
|
|
813108b9d8 | ||
|
|
95703b3af5 | ||
|
|
2ceec7c3d4 | ||
|
|
3985f1578e | ||
|
|
355ccb1e32 | ||
|
|
04d2c6f013 | ||
|
|
8d6a7101f6 | ||
|
|
abbd8473bb | ||
|
|
02b2103d94 | ||
|
|
85474ae381 | ||
|
|
9d3c401abb | ||
|
|
32a7d1a4a0 | ||
|
|
602cd5bd2e | ||
|
|
4b88b191e7 | ||
|
|
e449cd01ba | ||
|
|
b1db42acf4 | ||
|
|
fcbe8cf00b | ||
|
|
a1866c40f5 | ||
|
|
d78540f3cc | ||
|
|
7c1c04674c | ||
|
|
694aeab75c | ||
|
|
90aaf64b77 | ||
|
|
6eeabb1a1a | ||
|
|
77d58b82a0 | ||
|
|
a7af877e9e | ||
|
|
25092f56be | ||
|
|
e55f9dd21e | ||
|
|
6c10509705 | ||
|
|
bed2e253b2 | ||
|
|
c683ae9ed6 | ||
|
|
8ce1cebd07 | ||
|
|
9029b4b5c3 | ||
|
|
db0831d55b | ||
|
|
b3705073a3 | ||
|
|
755e8ce101 | ||
|
|
fd2ab47a16 | ||
|
|
9aafb7a743 | ||
|
|
aa1b20cf7b | ||
|
|
3cd2227c29 | ||
|
|
27cc83c03b | ||
|
|
cb6db3f3d8 | ||
|
|
9020e39a83 | ||
|
|
64509ca95d | ||
|
|
dafd62104b | ||
|
|
e09588e433 | ||
|
|
50e00192e8 | ||
|
|
e6d470f110 | ||
|
|
eeaf9a72e2 | ||
|
|
a4ad1bb0a9 | ||
|
|
e0086682b9 | ||
|
|
15ac2ade59 | ||
|
|
09e0d5c55e | ||
|
|
dfe855c0c9 | ||
|
|
a2996768fe | ||
|
|
a53b454d86 | ||
|
|
01557b599a | ||
|
|
65323b4169 | ||
|
|
cdef8b4852 | ||
|
|
8d72142390 | ||
|
|
172a0f0ac2 | ||
|
|
1022080be6 | ||
|
|
8031a55dbb | ||
|
|
1ddd29656e | ||
|
|
cb5c39a159 | ||
|
|
9ec510c742 | ||
|
|
aa80cf47b9 | ||
|
|
e4e1d42e09 | ||
|
|
ccc9a900ef | ||
|
|
c2a5ff0ae3 | ||
|
|
afa710dcb5 | ||
|
|
d0ed7593de | ||
|
|
830fc7f8cc | ||
|
|
88e8926a59 | ||
|
|
65e39b8e76 | ||
|
|
acd0c2188f | ||
|
|
6155cda66f | ||
|
|
f77f2f6e80 | ||
|
|
68d397a269 | ||
|
|
66ed57c834 | ||
|
|
8b42475d59 | ||
|
|
de9159d83c | ||
|
|
b66adfdf78 | ||
|
|
a4be5eb1d8 | ||
|
|
c8b6db4b84 | ||
|
|
7d3eef1fd9 | ||
|
|
6e93c5bc0d | ||
|
|
16b21b8bda | ||
|
|
a0e0039988 | ||
|
|
3970e91766 | ||
|
|
c4126edb79 | ||
|
|
589a6588da | ||
|
|
1ec4111b9f | ||
|
|
c00912015a | ||
|
|
0d19439982 | ||
|
|
12f790ab01 | ||
|
|
4e687c4fd8 | ||
|
|
ff960fda0e | ||
|
|
72e45b5cb1 | ||
|
|
b59e0490a2 | ||
|
|
94754f2047 | ||
|
|
ca335b7a65 | ||
|
|
8f9e6a519d | ||
|
|
da75968078 | ||
|
|
3800fb1ab0 | ||
|
|
a51bf6a4fa | ||
|
|
0be141188c | ||
|
|
6f2b985b73 | ||
|
|
8eba05308d | ||
|
|
ecab123a3a | ||
|
|
180f6c4519 | ||
|
|
dc4e362e10 | ||
|
|
7e78fdec04 | ||
|
|
e212ce23e3 | ||
|
|
0c7be8975f | ||
|
|
6cba95c148 | ||
|
|
f62d128621 | ||
|
|
91a86fd12f | ||
|
|
c819d494c3 | ||
|
|
b4853c5f67 | ||
|
|
c68334b421 | ||
|
|
613acc7b00 | ||
|
|
371fc4d68c | ||
|
|
7ef0ae12a1 | ||
|
|
dc34e8884c | ||
|
|
08d635322e | ||
|
|
bc0c1b9a3e | ||
|
|
c42ee926da | ||
|
|
c59d08080e | ||
|
|
94e0624024 | ||
|
|
e94a2c7c94 | ||
|
|
cd1ce32377 | ||
|
|
d1560ef09b | ||
|
|
6699d1e2f8 | ||
|
|
f1fbfa1b39 | ||
|
|
0d86717a9a | ||
|
|
528af27e4a | ||
|
|
fe69afaa56 | ||
|
|
31d7823498 | ||
|
|
ef6ec03161 | ||
|
|
68596be1b9 | ||
|
|
a48fe84971 | ||
|
|
bfbe7c1bf5 | ||
|
|
085ce1ff16 | ||
|
|
c81c0d9480 | ||
|
|
05b6bdb8f6 | ||
|
|
991d4f8859 | ||
|
|
22c75605ee | ||
|
|
f52faa8d14 | ||
|
|
8dd88a2780 | ||
|
|
7ad2234983 | ||
|
|
fb5c008fc5 | ||
|
|
f55b6bdc71 | ||
|
|
de9edb0695 | ||
|
|
2180e24bc1 | ||
|
|
749e64b101 | ||
|
|
b4eaa5f918 | ||
|
|
d4194954d3 | ||
|
|
6e221fc7d9 | ||
|
|
ea1285cd9f | ||
|
|
4e77fa100b | ||
|
|
babb76d90d | ||
|
|
f73153ed8d | ||
|
|
acfc1f56ee | ||
|
|
81773a6497 | ||
|
|
25cf751158 | ||
|
|
a9c8061c0c | ||
|
|
f53b1ee290 | ||
|
|
85b46ecff6 | ||
|
|
fd2753b95a | ||
|
|
768934e1cc | ||
|
|
ad7dee3e7f | ||
|
|
354c69bf4f | ||
|
|
52aa045635 | ||
|
|
1c8f92239d | ||
|
|
0c85de7839 | ||
|
|
d74d06d97b | ||
|
|
55351c5842 | ||
|
|
e67152ec14 | ||
|
|
287ff0a557 | ||
|
|
bf539d67ea | ||
|
|
69651453dd | ||
|
|
7f743ff3f1 | ||
|
|
a1b7c61b56 | ||
|
|
fbd1d53542 | ||
|
|
02adc74e2c | ||
|
|
3a452a3b2a | ||
|
|
91f8b3b505 | ||
|
|
ce09f2eb0f | ||
|
|
02d479b92d | ||
|
|
bce14c3d79 | ||
|
|
90cfcaac40 | ||
|
|
02563db2c7 | ||
|
|
cfa32a3986 | ||
|
|
ab4a6bc5b8 | ||
|
|
f01b8c849d | ||
|
|
6a46f50a35 | ||
|
|
468ed91ee3 | ||
|
|
b856e2120a | ||
|
|
fd7bfbea78 | ||
|
|
a74434e269 | ||
|
|
92be63ff5b | ||
|
|
4ce16b6509 | ||
|
|
2341667d04 | ||
|
|
89e7c8582e | ||
|
|
da4e4ecd23 | ||
|
|
c85648d43b | ||
|
|
c90331af98 | ||
|
|
e5461f6d4f | ||
|
|
ac82d6b27b | ||
|
|
01db974040 | ||
|
|
6ae6cf733d | ||
|
|
ad1d25d945 | ||
|
|
69ed47fc22 | ||
|
|
7c62654534 | ||
|
|
531e6c1a61 | ||
|
|
066f797ad4 | ||
|
|
9f0611f3d9 | ||
|
|
b0d2efd613 | ||
|
|
073283a433 | ||
|
|
5be04448ea | ||
|
|
5541b80179 | ||
|
|
35232b3650 | ||
|
|
f4ca3ea66b | ||
|
|
306efa17cc | ||
|
|
83107ad877 | ||
|
|
b0cf25bf5c | ||
|
|
6330424bcd | ||
|
|
5f3850bfa2 | ||
|
|
82e6658483 | ||
|
|
49f4add8d1 | ||
|
|
6ea96e79bd | ||
|
|
9ad31008a5 | ||
|
|
966f47a5b1 | ||
|
|
8711ece274 | ||
|
|
c0e2d56f55 | ||
|
|
73da5a35b8 | ||
|
|
2ba044ee4e | ||
|
|
d69180da71 | ||
|
|
b6e4cb9102 | ||
|
|
99d16fcb5a | ||
|
|
c8d0d765d1 | ||
|
|
96c318ee88 | ||
|
|
c1ac4d8261 | ||
|
|
e678df86e0 | ||
|
|
6aa124592d | ||
|
|
86b6b860dc | ||
|
|
53ee3a5f80 | ||
|
|
8af410b184 | ||
|
|
8cb3fe8831 | ||
|
|
2e92f3acad | ||
|
|
4e58094e4b | ||
|
|
1445340bba | ||
|
|
eb0eacd404 | ||
|
|
1ff3ab4b83 | ||
|
|
06841a4fde | ||
|
|
d57cd5acd7 | ||
|
|
b40cda78ee | ||
|
|
0a833f5f83 | ||
|
|
ef3bab16d3 | ||
|
|
3e826d0a5d | ||
|
|
bcd2c7d90b | ||
|
|
eda1b0b3fc | ||
|
|
bc7fe680c0 | ||
|
|
3ed748f2fd | ||
|
|
9634753efd | ||
|
|
31df4f0bb5 | ||
|
|
1673b8debc | ||
|
|
3d7ece91eb | ||
|
|
1004a83dae | ||
|
|
e7b5b82909 | ||
|
|
c17fddddcd | ||
|
|
2664a34d52 | ||
|
|
c9bf9691e3 | ||
|
|
ad5e5fed82 | ||
|
|
0e2fbe1c40 | ||
|
|
62e65d61f4 | ||
|
|
a237cbd963 | ||
|
|
78b12dc411 | ||
|
|
27457db5c3 | ||
|
|
e569fdd43c | ||
|
|
00a850500d | ||
|
|
25aa2f95a5 | ||
|
|
dc2ccc4fe8 | ||
|
|
cd5f260061 | ||
|
|
c47ba498a4 | ||
|
|
a6bf8e0eb7 | ||
|
|
930d4b3acf | ||
|
|
c481838179 | ||
|
|
963649998a | ||
|
|
dc4b950e8f | ||
|
|
5fea68a9b9 | ||
|
|
568a7ae16a | ||
|
|
f69d47587f | ||
|
|
7148767b6f | ||
|
|
c66b1646a6 | ||
|
|
bbcdc881cd | ||
|
|
07a78cf73d | ||
|
|
93ac2e3bc9 | ||
|
|
7e9614b8d1 | ||
|
|
7d704f966b | ||
|
|
43a2c6515d |
No files matched your search
@@ -15,6 +15,7 @@ RUN apt-get update && \
|
||||
build-essential \
|
||||
automake libtool \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# FROM debian:10-slim
|
||||
FROM python:3.7.12-slim-buster
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
automake libtool \
|
||||
pkg-config \
|
||||
librocksdb-dev
|
||||
# python3.7 \
|
||||
# python3-dev \
|
||||
# python3-pip \
|
||||
# python3-wheel \
|
||||
# python3-cffi \
|
||||
# python3-setuptools && \
|
||||
# update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
# rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install lbry-rocksdb
|
||||
RUN pip install uvloop
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@192.99.151.178:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -201,7 +201,7 @@ class AbstractBlob:
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash, added_on, is_mine)
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), added_on, blob_hash, is_mine)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
|
||||
@@ -12,8 +12,8 @@ class BlobInfo:
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self, blob_num: int, length: int, iv: str,
|
||||
blob_hash: typing.Optional[str] = None, added_on=0, is_mine=False):
|
||||
self, blob_num: int, length: int, iv: str, added_on,
|
||||
blob_hash: typing.Optional[str] = None, is_mine=False):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
|
||||
@@ -83,6 +83,8 @@ class BlobManager:
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
# check blobs that aren't set as finished but were seen on disk
|
||||
await self.ensure_completed_blobs_status(in_blobfiles_dir - to_add)
|
||||
if self.config.track_bandwidth:
|
||||
self.connection_manager.start()
|
||||
return True
|
||||
@@ -113,9 +115,18 @@ class BlobManager:
|
||||
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
|
||||
)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
async def ensure_completed_blobs_status(self, blob_hashes: typing.Iterable[str]):
|
||||
"""Ensures that completed blobs from a given list of blob hashes are set as 'finished' in the database."""
|
||||
to_add = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not self.is_blob_verified(blob_hash):
|
||||
continue
|
||||
blob = self.get_blob(blob_hash)
|
||||
to_add.append((blob.blob_hash, blob.length, blob.added_on, blob.is_mine))
|
||||
if len(to_add) > 500:
|
||||
await self.storage.add_blobs(*to_add, finished=True)
|
||||
to_add.clear()
|
||||
return await self.storage.add_blobs(*to_add, finished=True)
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
|
||||
@@ -36,31 +36,29 @@ class DiskSpaceManager:
|
||||
await self._clean(True)
|
||||
|
||||
async def _clean(self, is_network_blob=False):
|
||||
space_used_bytes = await self.get_space_used_bytes()
|
||||
space_used_mb = await self.get_space_used_mb(cached=False)
|
||||
if is_network_blob:
|
||||
space_used_bytes = space_used_bytes['network_storage']
|
||||
space_used_mb = space_used_mb['network_storage']
|
||||
else:
|
||||
space_used_bytes = space_used_bytes['content_storage'] + space_used_bytes['private_storage']
|
||||
space_used_mb = space_used_mb['content_storage'] + space_used_mb['private_storage']
|
||||
storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
|
||||
storage_limit = storage_limit_mb*1024*1024 if storage_limit_mb else None
|
||||
if self.analytics:
|
||||
asyncio.create_task(
|
||||
self.analytics.send_disk_space_used(space_used_bytes, storage_limit, is_network_blob)
|
||||
self.analytics.send_disk_space_used(space_used_mb, storage_limit_mb, is_network_blob)
|
||||
)
|
||||
if not storage_limit:
|
||||
return 0
|
||||
delete = []
|
||||
available = storage_limit - space_used_bytes
|
||||
if available > 0:
|
||||
available = storage_limit_mb - space_used_mb
|
||||
if storage_limit_mb == 0 if not is_network_blob else available >= 0:
|
||||
return 0
|
||||
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False, is_network_blob=is_network_blob):
|
||||
delete.append(blob_hash)
|
||||
available += file_size
|
||||
if available > 0:
|
||||
available += int(file_size/1024.0/1024.0)
|
||||
if available >= 0:
|
||||
break
|
||||
if delete:
|
||||
await self.db.stop_all_files()
|
||||
await self.blob_manager.delete_blobs(delete, delete_from_db=True)
|
||||
self._used_space_bytes = None
|
||||
return len(delete)
|
||||
|
||||
async def cleaning_loop(self):
|
||||
|
||||
@@ -97,21 +97,15 @@ class BlobDownloader:
|
||||
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
re_add: typing.Set['KademliaPeer'] = set()
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored:
|
||||
continue
|
||||
if peer in self.active_connections:
|
||||
if peer not in re_add:
|
||||
re_add.add(peer)
|
||||
if peer in self.active_connections or not self.should_race_continue(blob):
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
if not re_add:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
|
||||
+1
-1
@@ -622,7 +622,7 @@ class Config(CLIConfig):
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
"use.", 2
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
|
||||
+38
-16
@@ -1,6 +1,9 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
|
||||
from prometheus_client import Counter, Gauge
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.node import Node
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
@@ -9,24 +12,39 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
announcements_sent_metric = Counter(
|
||||
"announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node",
|
||||
labelnames=("peers", "error"),
|
||||
)
|
||||
announcement_queue_size_metric = Gauge(
|
||||
"announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.announce_queue: typing.List[str] = []
|
||||
self._done = asyncio.Event()
|
||||
self.announced = set()
|
||||
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
async def _run_consumer(self):
|
||||
while self.announce_queue:
|
||||
try:
|
||||
blob_hash = self.announce_queue.pop()
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
self.announcements_sent_metric.labels(peers=peers, error=False).inc()
|
||||
if peers > 4:
|
||||
self.announced.add(blob_hash)
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
self.announcements_sent_metric.labels(peers=0, error=True).inc()
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
@@ -37,17 +55,18 @@ class BlobAnnouncer:
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue))
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue) > 0:
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
await asyncio.gather(*[self._run_consumer() for _ in range(batch_size)], loop=self.loop)
|
||||
announced = list(filter(None, self.announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
self.announced.clear()
|
||||
self._done.set()
|
||||
self._done.clear()
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
@@ -56,3 +75,6 @@ class BlobAnnouncer:
|
||||
def stop(self):
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
|
||||
def wait(self):
|
||||
return self._done.wait()
|
||||
@@ -20,7 +20,6 @@ MAYBE_PING_DELAY = 300 # 5 minutes
|
||||
CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
|
||||
RPC_ID_LENGTH = 20
|
||||
PROTOCOL_VERSION = 1
|
||||
BOTTOM_OUT_LIMIT = 3
|
||||
MSG_SIZE_LIMIT = 1400
|
||||
|
||||
|
||||
|
||||
+4
-6
@@ -202,25 +202,23 @@ class Node:
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
|
||||
max_results: int = constants.K) -> IterativeNodeFinder:
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
key, max_results, None, shortlist)
|
||||
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = 40,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
key, max_results, None, shortlist)
|
||||
|
||||
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
|
||||
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
) -> typing.List['KademliaPeer']:
|
||||
peers = []
|
||||
async for iteration_peers in self.get_iterative_node_finder(
|
||||
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
|
||||
node_id, shortlist=shortlist, max_results=max_results):
|
||||
peers.extend(iteration_peers)
|
||||
distance = Distance(node_id)
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
|
||||
+4
-1
@@ -11,7 +11,7 @@ from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
CACHE_SIZE = 2048
|
||||
CACHE_SIZE = 16384
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -190,3 +190,6 @@ class KademliaPeer:
|
||||
|
||||
def compact_ip(self):
|
||||
return make_compact_ip(self.address)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.__class__.__name__}({self.node_id.hex()[:8]}@{self.address}:{self.udp_port}-{self.tcp_port})"
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
from itertools import chain
|
||||
from collections import defaultdict
|
||||
from collections import defaultdict, OrderedDict
|
||||
import typing
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -74,7 +74,7 @@ def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
class IterativeFinder:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
@@ -85,28 +85,22 @@ class IterativeFinder:
|
||||
self.protocol = protocol
|
||||
|
||||
self.key = key
|
||||
self.bottom_out_limit = bottom_out_limit
|
||||
self.max_results = max_results
|
||||
self.max_results = max(constants.K, max_results)
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.active: typing.Dict['KademliaPeer', int] = OrderedDict() # peer: distance, sorted
|
||||
self.contacted: typing.Set['KademliaPeer'] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.running_probes: typing.Dict['KademliaPeer', asyncio.Task] = {}
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
for peer in get_shortlist(routing_table, key, shortlist):
|
||||
if peer.node_id:
|
||||
self._add_active(peer)
|
||||
self._add_active(peer, force=True)
|
||||
else:
|
||||
# seed nodes
|
||||
self._schedule_probe(peer)
|
||||
@@ -138,15 +132,14 @@ class IterativeFinder:
|
||||
"""
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _add_active(self, peer):
|
||||
def _add_active(self, peer, force=False):
|
||||
if not force and self.peer_manager.peer_is_good(peer) is False:
|
||||
return
|
||||
if peer in self.contacted:
|
||||
return
|
||||
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
|
||||
self.active.add(peer)
|
||||
if self._is_closer(peer):
|
||||
self.prev_closest_peer = self.closest_peer
|
||||
self.closest_peer = peer
|
||||
self.active[peer] = self.distance(peer.node_id)
|
||||
self.active = OrderedDict(sorted(self.active.items(), key=lambda item: item[1]))
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
self._add_active(peer)
|
||||
@@ -158,33 +151,43 @@ class IterativeFinder:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
|
||||
peer.udp_port, address, udp_port)
|
||||
self.check_result_ready(response)
|
||||
self._log_state()
|
||||
|
||||
def _reset_closest(self, peer):
|
||||
if peer in self.active:
|
||||
del self.active[peer]
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
self._reset_closest(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
self._reset_closest(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
self._reset_closest(peer)
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
|
||||
async def _search_round(self):
|
||||
def _search_round(self):
|
||||
"""
|
||||
Send up to constants.alpha (5) probes to closest active peers
|
||||
"""
|
||||
|
||||
added = 0
|
||||
to_probe = list(self.active - self.contacted)
|
||||
to_probe.sort(key=lambda peer: self.distance(self.key))
|
||||
for peer in to_probe:
|
||||
if added >= constants.ALPHA:
|
||||
for index, peer in enumerate(self.active.keys()):
|
||||
if index == 0:
|
||||
log.debug("closest to probe: %s", peer.node_id.hex()[:8])
|
||||
if peer in self.contacted:
|
||||
continue
|
||||
if len(self.running_probes) >= constants.ALPHA:
|
||||
break
|
||||
if index > (constants.K + len(self.running_probes)):
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude:
|
||||
@@ -206,33 +209,22 @@ class IterativeFinder:
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
self.running_probes.pop(peer, None)
|
||||
if self.running:
|
||||
self._search_round()
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
self.running_probes[peer] = t
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
def _log_state(self):
|
||||
log.debug("[%s] check result: %i active nodes %i contacted",
|
||||
self.key.hex()[:8], len(self.active), len(self.contacted))
|
||||
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
self.loop.call_soon(self._search_round)
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
@@ -252,20 +244,19 @@ class IterativeFinder:
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
for task in chain(self.tasks, self.running_probes.values()):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, max_results, exclude,
|
||||
shortlist)
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
@@ -276,14 +267,14 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
self.put_result(self.active.keys(), finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [
|
||||
peer for peer in from_iter
|
||||
if peer not in self.yielded_peers
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
and self.peer_manager.peer_is_good(peer) is True # return only peers who answered
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:max(constants.K, self.max_results)]
|
||||
@@ -298,26 +289,16 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.active, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
self.bottom_out_count = 0
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
return self.put_result(self.active.keys(), finish=True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
# this tracks the index of the most recent page we requested from each peer
|
||||
@@ -362,23 +343,12 @@ class IterativeValueFinder(IterativeFinder):
|
||||
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
|
||||
for compact_addr in response.found_compact_addresses]
|
||||
to_yield = []
|
||||
self.bottom_out_count = 0
|
||||
for blob_peer in blob_peers:
|
||||
if blob_peer not in self.blob_peers:
|
||||
self.blob_peers.add(blob_peer)
|
||||
to_yield.append(blob_peer)
|
||||
if to_yield:
|
||||
# log.info("found %i new peers for blob", len(to_yield))
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
# if self.max_results and len(self.blob_peers) >= self.max_results:
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
@@ -6,6 +6,7 @@ import itertools
|
||||
|
||||
from prometheus_client import Gauge
|
||||
|
||||
from lbry import utils
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -70,9 +71,8 @@ class KBucket:
|
||||
if len(self.peers) < constants.K:
|
||||
self.peers.append(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").inc()
|
||||
if peer.node_id[0] == self._node_id[0]:
|
||||
bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).inc()
|
||||
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).inc()
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
@@ -140,9 +140,8 @@ class KBucket:
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
self.peers.remove(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").dec()
|
||||
if peer.node_id[0] == self._node_id[0]:
|
||||
bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).dec()
|
||||
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).dec()
|
||||
|
||||
def key_in_range(self, key: bytes) -> bool:
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
|
||||
@@ -381,6 +381,7 @@ class FileManagerComponent(Component):
|
||||
|
||||
|
||||
class BackgroundDownloaderComponent(Component):
|
||||
MIN_PREFIX_COLLIDING_BITS = 8
|
||||
component_name = BACKGROUND_DOWNLOADER_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT]
|
||||
|
||||
@@ -412,12 +413,18 @@ class BackgroundDownloaderComponent(Component):
|
||||
while True:
|
||||
self.space_available = await self.space_manager.get_free_space_mb(True)
|
||||
if not self.is_busy and self.space_available > 10:
|
||||
blob_hash = next((key.hex() for key in self.dht_node.stored_blob_hashes if
|
||||
key.hex() not in self.blob_manager.completed_blob_hashes), None)
|
||||
if blob_hash:
|
||||
self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash))
|
||||
self._download_next_close_blob_hash()
|
||||
await asyncio.sleep(self.download_loop_delay_seconds)
|
||||
|
||||
def _download_next_close_blob_hash(self):
|
||||
node_id = self.dht_node.protocol.node_id
|
||||
for blob_hash in self.dht_node.stored_blob_hashes:
|
||||
if blob_hash.hex() in self.blob_manager.completed_blob_hashes:
|
||||
continue
|
||||
if utils.get_colliding_prefix_bits(node_id, blob_hash) >= self.MIN_PREFIX_COLLIDING_BITS:
|
||||
self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash.hex()))
|
||||
return
|
||||
|
||||
async def start(self):
|
||||
self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT)
|
||||
if not self.component_manager.has_component(DHT_COMPONENT):
|
||||
|
||||
@@ -8,6 +8,7 @@ import time
|
||||
import inspect
|
||||
import typing
|
||||
import random
|
||||
import hashlib
|
||||
import tracemalloc
|
||||
from decimal import Decimal
|
||||
from urllib.parse import urlencode, quote
|
||||
@@ -16,6 +17,7 @@ from binascii import hexlify, unhexlify
|
||||
from traceback import format_exc
|
||||
from functools import wraps, partial
|
||||
|
||||
import ecdsa
|
||||
import base58
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter
|
||||
@@ -27,7 +29,6 @@ from lbry.wallet import (
|
||||
)
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
|
||||
from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES
|
||||
from lbry.wallet.bip32 import PrivateKey
|
||||
|
||||
from lbry import utils
|
||||
from lbry.conf import Config, Setting, NOT_SET
|
||||
@@ -37,7 +38,8 @@ from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import (
|
||||
DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError,
|
||||
CommandDoesNotExistError, BaseError, WalletNotFoundError, WalletAlreadyLoadedError, WalletAlreadyExistsError,
|
||||
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError
|
||||
ConflictingInputValueError, AlreadyPurchasedError, PrivateKeyNotFoundError, InputStringIsBlankError,
|
||||
InputValueError
|
||||
)
|
||||
from lbry.extras import system_info
|
||||
from lbry.extras.daemon import analytics
|
||||
@@ -2705,13 +2707,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
name, claim, amount, claim_address, funding_accounts, funding_accounts[0]
|
||||
)
|
||||
txo = tx.outputs[0]
|
||||
txo.set_channel_private_key(
|
||||
await funding_accounts[0].generate_channel_private_key()
|
||||
)
|
||||
await txo.generate_channel_private_key()
|
||||
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
account.add_channel_private_key(txo.private_key)
|
||||
wallet.save()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
@@ -2860,9 +2861,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
new_txo = tx.outputs[0]
|
||||
|
||||
if new_signing_key:
|
||||
new_txo.set_channel_private_key(
|
||||
await funding_accounts[0].generate_channel_private_key()
|
||||
)
|
||||
await new_txo.generate_channel_private_key()
|
||||
else:
|
||||
new_txo.private_key = old_txo.private_key
|
||||
|
||||
@@ -2871,6 +2870,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
account.add_channel_private_key(new_txo.private_key)
|
||||
wallet.save()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
@@ -3042,7 +3042,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'channel_id': channel.claim_id,
|
||||
'holding_address': address,
|
||||
'holding_public_key': public_key.extended_key_string(),
|
||||
'signing_private_key': channel.private_key.signing_key.to_pem().decode()
|
||||
'signing_private_key': channel.private_key.to_pem().decode()
|
||||
}
|
||||
return base58.b58encode(json.dumps(export, separators=(',', ':')))
|
||||
|
||||
@@ -3065,14 +3065,15 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
decoded = base58.b58decode(channel_data)
|
||||
data = json.loads(decoded)
|
||||
channel_private_key = PrivateKey.from_pem(
|
||||
self.ledger, data['signing_private_key']
|
||||
channel_private_key = ecdsa.SigningKey.from_pem(
|
||||
data['signing_private_key'], hashfunc=hashlib.sha256
|
||||
)
|
||||
public_key_der = channel_private_key.get_verifying_key().to_der()
|
||||
|
||||
# check that the holding_address hasn't changed since the export was made
|
||||
holding_address = data['holding_address']
|
||||
channels, _, _, _ = await self.ledger.claim_search(
|
||||
wallet.accounts, public_key_id=channel_private_key.address
|
||||
wallet.accounts, public_key_id=self.ledger.public_key_to_address(public_key_der)
|
||||
)
|
||||
if channels and channels[0].get_address(self.ledger) != holding_address:
|
||||
holding_address = channels[0].get_address(self.ledger)
|
||||
@@ -3614,15 +3615,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
if len(existing_claims) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
raise InputValueError(
|
||||
f"Can't find the stream '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
|
||||
old_txo = existing_claims[0]
|
||||
if not old_txo.claim.is_stream:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream claim."
|
||||
if not old_txo.claim.is_stream and not old_txo.claim.is_repost:
|
||||
# in principle it should work with any type of claim, but its safer to
|
||||
# limit it to ones we know won't be broken. in the future we can expand
|
||||
# this if we have a test case for e.g. channel or support claims
|
||||
raise InputValueError(
|
||||
f"A claim with id '{claim_id}' was found but it is not a stream or repost claim."
|
||||
)
|
||||
|
||||
if bid is not None:
|
||||
@@ -3653,28 +3656,32 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if replace:
|
||||
claim = Claim()
|
||||
if old_txo.claim.stream.has_source:
|
||||
claim.stream.message.source.CopyFrom(
|
||||
old_txo.claim.stream.message.source
|
||||
)
|
||||
stream_type = old_txo.claim.stream.stream_type
|
||||
if stream_type:
|
||||
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
|
||||
new_stream_type = getattr(claim.stream.message, stream_type)
|
||||
new_stream_type.CopyFrom(old_stream_type)
|
||||
claim.stream.update(file_path=file_path, **kwargs)
|
||||
if old_txo.claim.is_stream:
|
||||
if old_txo.claim.stream.has_source:
|
||||
claim.stream.message.source.CopyFrom(
|
||||
old_txo.claim.stream.message.source
|
||||
)
|
||||
stream_type = old_txo.claim.stream.stream_type
|
||||
if stream_type:
|
||||
old_stream_type = getattr(old_txo.claim.stream.message, stream_type)
|
||||
new_stream_type = getattr(claim.stream.message, stream_type)
|
||||
new_stream_type.CopyFrom(old_stream_type)
|
||||
else:
|
||||
claim = Claim.from_bytes(old_txo.claim.to_bytes())
|
||||
|
||||
if old_txo.claim.is_stream:
|
||||
claim.stream.update(file_path=file_path, **kwargs)
|
||||
|
||||
if clear_channel:
|
||||
claim.clear_signature()
|
||||
tx = await Transaction.claim_update(
|
||||
old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0],
|
||||
channel if not clear_channel else None
|
||||
)
|
||||
|
||||
new_txo = tx.outputs[0]
|
||||
stream_hash = None
|
||||
if not preview:
|
||||
if not preview and old_txo.claim.is_stream:
|
||||
old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash)
|
||||
old_stream = old_stream[0] if old_stream else None
|
||||
if file_path is not None:
|
||||
@@ -4878,20 +4885,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None, page=None, page_size=None):
|
||||
async def jsonrpc_peer_list(self, blob_hash, page=None, page_size=None):
|
||||
"""
|
||||
Get peers for blob hash
|
||||
|
||||
Usage:
|
||||
peer_list (<blob_hash> | --blob_hash=<blob_hash>)
|
||||
[<search_bottom_out_limit> | --search_bottom_out_limit=<search_bottom_out_limit>]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--blob_hash=<blob_hash> : (str) find available peers for this blob hash
|
||||
--search_bottom_out_limit=<search_bottom_out_limit> : (int) the number of search probes in a row
|
||||
that don't find any new peers
|
||||
before giving up and returning
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
|
||||
@@ -4903,13 +4906,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("invalid blob hash")
|
||||
if search_bottom_out_limit is not None:
|
||||
search_bottom_out_limit = int(search_bottom_out_limit)
|
||||
if search_bottom_out_limit <= 0:
|
||||
# TODO: use error from lbry.error
|
||||
raise Exception("invalid bottom out limit")
|
||||
else:
|
||||
search_bottom_out_limit = 4
|
||||
peers = []
|
||||
peer_q = asyncio.Queue(loop=self.component_manager.loop)
|
||||
await self.dht_node._peers_for_value_producer(blob_hash, peer_q)
|
||||
|
||||
@@ -10,7 +10,7 @@ from lbry.schema.claim import Claim
|
||||
from lbry.schema.support import Support
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PublicKey
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
|
||||
@@ -138,7 +138,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, Support):
|
||||
return obj.to_dict()
|
||||
if isinstance(obj, PublicKey):
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
|
||||
@@ -37,6 +37,8 @@ def migrate_db(conf, start, end):
|
||||
from .migrate13to14 import do_migration
|
||||
elif current == 14:
|
||||
from .migrate14to15 import do_migration
|
||||
elif current == 15:
|
||||
from .migrate15to16 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
update blob set should_announce=0
|
||||
where should_announce=1 and
|
||||
blob.blob_hash in (select stream_blob.blob_hash from stream_blob where position=0);
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -20,7 +20,7 @@ def do_migration(conf):
|
||||
"left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
|
||||
blobs_by_stream = {}
|
||||
for stream_hash, position, iv, blob_hash, blob_length in blobs:
|
||||
blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
|
||||
blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, 0, blob_hash))
|
||||
|
||||
for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
|
||||
sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename,
|
||||
|
||||
@@ -187,8 +187,8 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
).fetchall()
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
"update blob set should_announce=1 where blob_hash in (?)",
|
||||
(sd_blob.blob_hash,)
|
||||
).fetchall()
|
||||
|
||||
|
||||
@@ -449,7 +449,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return await self.db.execute_fetchall(
|
||||
"select blob.blob_hash, blob.blob_length, blob.added_on "
|
||||
"from blob left join stream_blob using (blob_hash) "
|
||||
"where stream_blob.stream_hash is null and blob.is_mine=? order by blob.added_on asc",
|
||||
"where stream_blob.stream_hash is null and blob.is_mine=? "
|
||||
"order by blob.blob_length desc, blob.added_on asc",
|
||||
(is_mine,)
|
||||
)
|
||||
|
||||
@@ -462,7 +463,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
content_blobs = await self.db.execute_fetchall(
|
||||
"select blob.blob_hash, blob.blob_length, blob.added_on "
|
||||
"from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
|
||||
"cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc",
|
||||
"cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc, blob.blob_length asc",
|
||||
(is_mine,)
|
||||
)
|
||||
return content_blobs + sd_blobs
|
||||
@@ -479,7 +480,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
coalesce(sum(case when
|
||||
is_mine=1
|
||||
then blob_length else 0 end), 0) as private_storage
|
||||
from blob left join stream_blob using (blob_hash)
|
||||
from blob left join stream_blob using (blob_hash) where blob_hash not in (select sd_hash from stream)
|
||||
""")
|
||||
return {
|
||||
'network_storage': network_size,
|
||||
@@ -531,7 +532,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def _get_blobs_for_stream(transaction):
|
||||
crypt_blob_infos = []
|
||||
stream_blobs = transaction.execute(
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=? "
|
||||
"select s.blob_hash, s.position, s.iv, b.added_on "
|
||||
"from stream_blob s left outer join blob b on b.blob_hash=s.blob_hash where stream_hash=? "
|
||||
"order by position asc", (stream_hash, )
|
||||
).fetchall()
|
||||
if only_completed:
|
||||
@@ -551,9 +553,10 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for blob_hash, length in lengths:
|
||||
blob_length_dict[blob_hash] = length
|
||||
|
||||
for blob_hash, position, iv in stream_blobs:
|
||||
current_time = time.time()
|
||||
for blob_hash, position, iv, added_on in stream_blobs:
|
||||
blob_length = blob_length_dict.get(blob_hash, 0)
|
||||
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
|
||||
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, added_on or current_time, blob_hash))
|
||||
if not blob_hash:
|
||||
break
|
||||
return crypt_blob_infos
|
||||
|
||||
@@ -2,9 +2,6 @@ import logging
|
||||
from typing import List
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from asn1crypto.keys import PublicKeyInfo
|
||||
from coincurve import PublicKey as cPublicKey
|
||||
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from google.protobuf.message import DecodeError
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
@@ -349,7 +346,7 @@ class Channel(BaseClaim):
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.public_key_bytes).decode()
|
||||
return hexlify(self.message.public_key).decode()
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
@@ -357,11 +354,7 @@ class Channel(BaseClaim):
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
if len(self.message.public_key) == 33:
|
||||
return self.message.public_key
|
||||
public_key_info = PublicKeyInfo.load(self.message.public_key)
|
||||
public_key = cPublicKey(public_key_info.native['public_key'])
|
||||
return public_key.format(compressed=True)
|
||||
return self.message.public_key
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import filetype
|
||||
import logging
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
@@ -166,10 +168,38 @@ types_map = {
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
# maps detected extensions to the possible analogs
|
||||
# i.e. .cbz file is actually a .zip
|
||||
synonyms_map = {
|
||||
'.zip': ['.cbz'],
|
||||
'.rar': ['.cbr'],
|
||||
'.ar': ['.a']
|
||||
}
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
extension = ext.strip().lower()
|
||||
|
||||
try:
|
||||
kind = filetype.guess(path)
|
||||
if kind:
|
||||
real_extension = f".{kind.extension}"
|
||||
|
||||
if extension != real_extension:
|
||||
if extension:
|
||||
log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
|
||||
else:
|
||||
log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
|
||||
|
||||
if extension not in synonyms_map.get(real_extension, []):
|
||||
extension = real_extension
|
||||
|
||||
except OSError as error:
|
||||
pass
|
||||
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
|
||||
@@ -194,12 +194,13 @@ class StreamDescriptor:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
|
||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||
added_on = time.time()
|
||||
descriptor = cls(
|
||||
loop, blob_dir,
|
||||
binascii.unhexlify(decoded['stream_name']).decode(),
|
||||
decoded['key'],
|
||||
binascii.unhexlify(decoded['suggested_file_name']).decode(),
|
||||
[BlobInfo(info['blob_num'], info['length'], info['iv'], info.get('blob_hash'))
|
||||
[BlobInfo(info['blob_num'], info['length'], info['iv'], added_on, info.get('blob_hash'))
|
||||
for info in decoded['blobs']],
|
||||
decoded['stream_hash'],
|
||||
blob.blob_hash
|
||||
@@ -266,7 +267,7 @@ class StreamDescriptor:
|
||||
blobs.append(blob_info)
|
||||
blobs.append(
|
||||
# add the stream terminator
|
||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), None, added_on, True)
|
||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), added_on, None, True)
|
||||
)
|
||||
file_name = os.path.basename(file_path)
|
||||
suggested_file_name = sanitize_file_name(file_name)
|
||||
|
||||
@@ -98,10 +98,6 @@ class StreamDownloader:
|
||||
if not self.descriptor:
|
||||
await self.load_descriptor(connection_id)
|
||||
|
||||
# add the head blob to the peer search
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to peer search for stream %s", self.sd_hash)
|
||||
|
||||
if not await self.blob_manager.storage.stream_exists(self.sd_hash) and save_stream:
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
|
||||
|
||||
@@ -70,6 +70,7 @@ class StreamManager(SourceManager):
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
to_check = []
|
||||
|
||||
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str,
|
||||
@@ -82,6 +83,7 @@ class StreamManager(SourceManager):
|
||||
if not descriptor:
|
||||
return
|
||||
to_restore.append((descriptor, sd_blob, content_fee))
|
||||
to_check.extend([sd_blob.blob_hash] + [blob.blob_hash for blob in descriptor.blobs[:-1]])
|
||||
|
||||
await asyncio.gather(*[
|
||||
recover_stream(
|
||||
@@ -93,6 +95,8 @@ class StreamManager(SourceManager):
|
||||
|
||||
if to_restore:
|
||||
await self.storage.recover_streams(to_restore, self.config.download_dir)
|
||||
if to_check:
|
||||
await self.blob_manager.ensure_completed_blobs_status(to_check)
|
||||
|
||||
# if self.blob_manager._save_blobs:
|
||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
+22
-56
@@ -17,10 +17,8 @@ from functools import partial
|
||||
from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.util import satoshis_to_coins
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
from lbry.wallet.orchstr8 import Conductor
|
||||
from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode, HubNode
|
||||
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
@@ -204,7 +202,13 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
|
||||
def add_timeout(self):
|
||||
if self.TIMEOUT:
|
||||
self.loop.call_later(self.TIMEOUT, self.cancel)
|
||||
self.loop.call_later(self.TIMEOUT, self.check_timeout, time())
|
||||
|
||||
def check_timeout(self, started):
|
||||
if time() - started >= self.TIMEOUT:
|
||||
self.cancel()
|
||||
else:
|
||||
self.loop.call_later(self.TIMEOUT, self.check_timeout, started)
|
||||
|
||||
|
||||
class AdvanceTimeTestCase(AsyncioTestCase):
|
||||
@@ -230,7 +234,7 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.conductor: Optional[Conductor] = None
|
||||
self.blockchain: Optional[LBCWalletNode] = None
|
||||
self.blockchain: Optional[BlockchainNode] = None
|
||||
self.hub: Optional[HubNode] = None
|
||||
self.wallet_node: Optional[WalletNode] = None
|
||||
self.manager: Optional[WalletManager] = None
|
||||
@@ -240,17 +244,15 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.conductor = Conductor(seed=self.SEED)
|
||||
await self.conductor.start_lbcd()
|
||||
self.addCleanup(self.conductor.stop_lbcd)
|
||||
await self.conductor.start_lbcwallet()
|
||||
self.addCleanup(self.conductor.stop_lbcwallet)
|
||||
await self.conductor.start_blockchain()
|
||||
self.addCleanup(self.conductor.stop_blockchain)
|
||||
await self.conductor.start_spv()
|
||||
self.addCleanup(self.conductor.stop_spv)
|
||||
await self.conductor.start_wallet()
|
||||
self.addCleanup(self.conductor.stop_wallet)
|
||||
await self.conductor.start_hub()
|
||||
self.addCleanup(self.conductor.stop_hub)
|
||||
self.blockchain = self.conductor.lbcwallet_node
|
||||
self.blockchain = self.conductor.blockchain_node
|
||||
self.hub = self.conductor.hub_node
|
||||
self.wallet_node = self.conductor.wallet_node
|
||||
self.manager = self.wallet_node.manager
|
||||
@@ -265,13 +267,6 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
def broadcast(self, tx):
|
||||
return self.ledger.broadcast(tx)
|
||||
|
||||
async def broadcast_and_confirm(self, tx, ledger=None):
|
||||
ledger = ledger or self.ledger
|
||||
notifications = asyncio.create_task(ledger.wait(tx))
|
||||
await ledger.broadcast(tx)
|
||||
await notifications
|
||||
await self.generate_and_wait(1, [tx.id], ledger)
|
||||
|
||||
async def on_header(self, height):
|
||||
if self.ledger.headers.height < height:
|
||||
await self.ledger.on_header.where(
|
||||
@@ -279,29 +274,11 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
)
|
||||
return True
|
||||
|
||||
async def send_to_address_and_wait(self, address, amount, blocks_to_generate=0, ledger=None):
|
||||
tx_watch = []
|
||||
txid = None
|
||||
done = False
|
||||
watcher = (ledger or self.ledger).on_transaction.where(
|
||||
lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id)
|
||||
def on_transaction_id(self, txid, ledger=None):
|
||||
return (ledger or self.ledger).on_transaction.where(
|
||||
lambda e: e.tx.id == txid
|
||||
)
|
||||
|
||||
txid = await self.blockchain.send_to_address(address, amount)
|
||||
done = txid in tx_watch
|
||||
await watcher
|
||||
|
||||
await self.generate_and_wait(blocks_to_generate, [txid], ledger)
|
||||
return txid
|
||||
|
||||
async def generate_and_wait(self, blocks_to_generate, txids, ledger=None):
|
||||
if blocks_to_generate > 0:
|
||||
watcher = (ledger or self.ledger).on_transaction.where(
|
||||
lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1] # multi-statement lambda
|
||||
)
|
||||
await self.blockchain.generate(blocks_to_generate)
|
||||
await watcher
|
||||
|
||||
def on_address_update(self, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.address == address
|
||||
@@ -372,19 +349,20 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.skip_libtorrent = True
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbry.stream').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY)
|
||||
|
||||
await super().asyncSetUp()
|
||||
|
||||
self.daemon = await self.add_daemon(self.wallet_node)
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
await self.send_to_address_and_wait(address, 10, 6)
|
||||
sendtxid = await self.blockchain.send_to_address(address, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(5)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
@@ -481,8 +459,9 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
async def confirm_tx(self, txid, ledger=None):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
# actually, if it's in the mempool or in the block we're fine
|
||||
await self.generate_and_wait(1, [txid], ledger=ledger)
|
||||
await self.on_transaction_id(txid, ledger)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid, ledger)
|
||||
return txid
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
@@ -533,19 +512,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
return self.sout(tx)
|
||||
return tx
|
||||
|
||||
async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False):
|
||||
account = (daemon or self.daemon).wallet_manager.default_account
|
||||
claim_address = await account.receiving.get_or_create_usable_address()
|
||||
claim = Claim()
|
||||
claim.channel.public_key_bytes = pubkey_bytes
|
||||
tx = await Transaction.claim_create(
|
||||
name, claim, lbc_to_dewies(price),
|
||||
claim_address, [self.account], self.account
|
||||
)
|
||||
await tx.sign([self.account])
|
||||
await (daemon or self.daemon).broadcast_or_release(tx, blocking)
|
||||
return self.sout(tx)
|
||||
|
||||
def create_upload_file(self, data, prefix=None, suffix=None):
|
||||
file_path = tempfile.mktemp(prefix=prefix or "tmp", suffix=suffix or "", dir=self.daemon.conf.upload_dir)
|
||||
with open(file_path, 'w+b') as file:
|
||||
|
||||
@@ -474,3 +474,18 @@ class LockWithMetrics(asyncio.Lock):
|
||||
return super().release()
|
||||
finally:
|
||||
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||
|
||||
|
||||
def get_colliding_prefix_bits(first_value: bytes, second_value: bytes):
|
||||
"""
|
||||
Calculates the amount of colliding prefix bits between <first_value> and <second_value>.
|
||||
This is given by the amount of bits that are the same until the first different one (via XOR),
|
||||
starting from the most significant bit to the least significant bit.
|
||||
:param first_value: first value to compare, bigger than size.
|
||||
:param second_value: second value to compare, bigger than size.
|
||||
:return: amount of prefix colliding bits.
|
||||
"""
|
||||
assert len(first_value) == len(second_value), "length should be the same"
|
||||
size = len(first_value) * 8
|
||||
first_value, second_value = int.from_bytes(first_value, "big"), int.from_bytes(second_value, "big")
|
||||
return size - (first_value ^ second_value).bit_length()
|
||||
+6
-11
@@ -1,13 +1,8 @@
|
||||
__lbcd__ = 'lbcd'
|
||||
__lbcctl__ = 'lbcctl'
|
||||
__lbcwallet__ = 'lbcwallet'
|
||||
__lbcd_url__ = (
|
||||
'https://github.com/lbryio/lbcd/releases/download/' +
|
||||
'v0.22.200-beta/lbcd_0.22.200-beta_TARGET_PLATFORM.tar.gz'
|
||||
)
|
||||
__lbcwallet_url__ = (
|
||||
'https://github.com/lbryio/lbcwallet/releases/download/' +
|
||||
'v0.13.100-alpha-rc2/lbcwallet_0.13.100-alpha-rc2_TARGET_PLATFORM.tar.gz'
|
||||
__node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
@@ -15,7 +10,7 @@ from .wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DI
|
||||
from .manager import WalletManager
|
||||
from .network import Network
|
||||
from .ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent
|
||||
from .account import Account, AddressManager, SingleKey, HierarchicalDeterministic, DeterministicChannelKeyManager
|
||||
from .account import Account, AddressManager, SingleKey, HierarchicalDeterministic
|
||||
from .transaction import Transaction, Output, Input
|
||||
from .script import OutputScript, InputScript
|
||||
from .database import SQLiteMixin, Database
|
||||
|
||||
+33
-65
@@ -9,10 +9,11 @@ from hashlib import sha256
|
||||
from string import hexdigits
|
||||
from typing import Type, Dict, Tuple, Optional, Any, List
|
||||
|
||||
import ecdsa
|
||||
from lbry.error import InvalidPasswordError
|
||||
from lbry.crypto.crypt import aes_encrypt, aes_decrypt
|
||||
|
||||
from .bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string
|
||||
from .bip32 import PrivateKey, PubKey, from_extended_key_string
|
||||
from .mnemonic import Mnemonic
|
||||
from .constants import COIN, TXO_TYPES
|
||||
from .transaction import Transaction, Input, Output
|
||||
@@ -33,44 +34,6 @@ def validate_claim_id(claim_id):
|
||||
raise Exception("Claim id is not hex encoded")
|
||||
|
||||
|
||||
class DeterministicChannelKeyManager:
|
||||
|
||||
def __init__(self, account: 'Account'):
|
||||
self.account = account
|
||||
self.last_known = 0
|
||||
self.cache = {}
|
||||
self.private_key: Optional[PrivateKey] = None
|
||||
if account.private_key is not None:
|
||||
self.private_key = account.private_key.child(KeyPath.CHANNEL)
|
||||
|
||||
def maybe_generate_deterministic_key_for_channel(self, txo):
|
||||
if self.private_key is None:
|
||||
return
|
||||
next_private_key = self.private_key.child(self.last_known)
|
||||
public_key = next_private_key.public_key
|
||||
public_key_bytes = public_key.pubkey_bytes
|
||||
if txo.claim.channel.public_key_bytes == public_key_bytes:
|
||||
self.cache[public_key.address] = next_private_key
|
||||
self.last_known += 1
|
||||
|
||||
async def ensure_cache_primed(self):
|
||||
if self.private_key is not None:
|
||||
await self.generate_next_key()
|
||||
|
||||
async def generate_next_key(self) -> PrivateKey:
|
||||
db = self.account.ledger.db
|
||||
while True:
|
||||
next_private_key = self.private_key.child(self.last_known)
|
||||
public_key = next_private_key.public_key
|
||||
self.cache[public_key.address] = next_private_key
|
||||
if not await db.is_channel_key_used(self.account, public_key):
|
||||
return next_private_key
|
||||
self.last_known += 1
|
||||
|
||||
def get_private_key_from_pubkey_hash(self, pubkey_hash) -> PrivateKey:
|
||||
return self.cache.get(pubkey_hash)
|
||||
|
||||
|
||||
class AddressManager:
|
||||
|
||||
name: str
|
||||
@@ -116,7 +79,7 @@ class AddressManager:
|
||||
def get_private_key(self, index: int) -> PrivateKey:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_public_key(self, index: int) -> PublicKey:
|
||||
def get_public_key(self, index: int) -> PubKey:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_max_gap(self):
|
||||
@@ -156,8 +119,8 @@ class HierarchicalDeterministic(AddressManager):
|
||||
@classmethod
|
||||
def from_dict(cls, account: 'Account', d: dict) -> Tuple[AddressManager, AddressManager]:
|
||||
return (
|
||||
cls(account, KeyPath.RECEIVE, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
|
||||
cls(account, KeyPath.CHANGE, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
|
||||
cls(account, 0, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
|
||||
cls(account, 1, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
|
||||
)
|
||||
|
||||
def merge(self, d: dict):
|
||||
@@ -170,7 +133,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
def get_private_key(self, index: int) -> PrivateKey:
|
||||
return self.account.private_key.child(self.chain_number).child(index)
|
||||
|
||||
def get_public_key(self, index: int) -> PublicKey:
|
||||
def get_public_key(self, index: int) -> PubKey:
|
||||
return self.account.public_key.child(self.chain_number).child(index)
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
@@ -230,7 +193,7 @@ class SingleKey(AddressManager):
|
||||
@classmethod
|
||||
def from_dict(cls, account: 'Account', d: dict) \
|
||||
-> Tuple[AddressManager, AddressManager]:
|
||||
same_address_manager = cls(account, account.public_key, KeyPath.RECEIVE)
|
||||
same_address_manager = cls(account, account.public_key, 0)
|
||||
return same_address_manager, same_address_manager
|
||||
|
||||
def to_dict_instance(self):
|
||||
@@ -239,7 +202,7 @@ class SingleKey(AddressManager):
|
||||
def get_private_key(self, index: int) -> PrivateKey:
|
||||
return self.account.private_key
|
||||
|
||||
def get_public_key(self, index: int) -> PublicKey:
|
||||
def get_public_key(self, index: int) -> PubKey:
|
||||
return self.account.public_key
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
@@ -261,6 +224,9 @@ class SingleKey(AddressManager):
|
||||
|
||||
class Account:
|
||||
|
||||
mnemonic_class = Mnemonic
|
||||
private_key_class = PrivateKey
|
||||
public_key_class = PubKey
|
||||
address_generators: Dict[str, Type[AddressManager]] = {
|
||||
SingleKey.name: SingleKey,
|
||||
HierarchicalDeterministic.name: HierarchicalDeterministic,
|
||||
@@ -268,7 +234,7 @@ class Account:
|
||||
|
||||
def __init__(self, ledger: 'Ledger', wallet: 'Wallet', name: str,
|
||||
seed: str, private_key_string: str, encrypted: bool,
|
||||
private_key: Optional[PrivateKey], public_key: PublicKey,
|
||||
private_key: Optional[PrivateKey], public_key: PubKey,
|
||||
address_generator: dict, modified_on: float, channel_keys: dict) -> None:
|
||||
self.ledger = ledger
|
||||
self.wallet = wallet
|
||||
@@ -279,14 +245,13 @@ class Account:
|
||||
self.private_key_string = private_key_string
|
||||
self.init_vectors: Dict[str, bytes] = {}
|
||||
self.encrypted = encrypted
|
||||
self.private_key: Optional[PrivateKey] = private_key
|
||||
self.public_key: PublicKey = public_key
|
||||
self.private_key = private_key
|
||||
self.public_key = public_key
|
||||
generator_name = address_generator.get('name', HierarchicalDeterministic.name)
|
||||
self.address_generator = self.address_generators[generator_name]
|
||||
self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
|
||||
self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)}
|
||||
self.channel_keys = channel_keys
|
||||
self.deterministic_channel_keys = DeterministicChannelKeyManager(self)
|
||||
ledger.add_account(self)
|
||||
wallet.add_account(self)
|
||||
|
||||
@@ -301,19 +266,19 @@ class Account:
|
||||
name: str = None, address_generator: dict = None):
|
||||
return cls.from_dict(ledger, wallet, {
|
||||
'name': name,
|
||||
'seed': Mnemonic().make_seed(),
|
||||
'seed': cls.mnemonic_class().make_seed(),
|
||||
'address_generator': address_generator or {}
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def get_private_key_from_seed(cls, ledger: 'Ledger', seed: str, password: str):
|
||||
return PrivateKey.from_seed(
|
||||
ledger, Mnemonic.mnemonic_to_seed(seed, password or 'lbryum')
|
||||
return cls.private_key_class.from_seed(
|
||||
ledger, cls.mnemonic_class.mnemonic_to_seed(seed, password or 'lbryum')
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def keys_from_dict(cls, ledger: 'Ledger', d: dict) \
|
||||
-> Tuple[str, Optional[PrivateKey], PublicKey]:
|
||||
-> Tuple[str, Optional[PrivateKey], PubKey]:
|
||||
seed = d.get('seed', '')
|
||||
private_key_string = d.get('private_key', '')
|
||||
private_key = None
|
||||
@@ -484,7 +449,7 @@ class Account:
|
||||
assert not self.encrypted, "Cannot get private key on encrypted wallet account."
|
||||
return self.address_managers[chain].get_private_key(index)
|
||||
|
||||
def get_public_key(self, chain: int, index: int) -> PublicKey:
|
||||
def get_public_key(self, chain: int, index: int) -> PubKey:
|
||||
return self.address_managers[chain].get_public_key(index)
|
||||
|
||||
def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
|
||||
@@ -555,30 +520,33 @@ class Account:
|
||||
|
||||
return tx
|
||||
|
||||
async def generate_channel_private_key(self):
|
||||
return await self.deterministic_channel_keys.generate_next_key()
|
||||
def add_channel_private_key(self, private_key):
|
||||
public_key_bytes = private_key.get_verifying_key().to_der()
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
|
||||
|
||||
def add_channel_private_key(self, private_key: PrivateKey):
|
||||
self.channel_keys[private_key.address] = private_key.to_pem().decode()
|
||||
|
||||
async def get_channel_private_key(self, public_key_bytes) -> PrivateKey:
|
||||
async def get_channel_private_key(self, public_key_bytes):
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
|
||||
if private_key_pem:
|
||||
return PrivateKey.from_pem(self.ledger, private_key_pem)
|
||||
return self.deterministic_channel_keys.get_private_key_from_pubkey_hash(channel_pubkey_hash)
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None, ecdsa.SigningKey.from_pem, private_key_pem, sha256
|
||||
)
|
||||
|
||||
async def maybe_migrate_certificates(self):
|
||||
def to_der(private_key_pem):
|
||||
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256).get_verifying_key().to_der()
|
||||
|
||||
if not self.channel_keys:
|
||||
return
|
||||
channel_keys = {}
|
||||
for private_key_pem in self.channel_keys.values():
|
||||
if not isinstance(private_key_pem, str):
|
||||
continue
|
||||
if not private_key_pem.startswith("-----BEGIN"):
|
||||
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
|
||||
continue
|
||||
private_key = PrivateKey.from_pem(self.ledger, private_key_pem)
|
||||
channel_keys[private_key.address] = private_key_pem
|
||||
public_key_der = await asyncio.get_event_loop().run_in_executor(None, to_der, private_key_pem)
|
||||
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
|
||||
if self.channel_keys != channel_keys:
|
||||
self.channel_keys = channel_keys
|
||||
self.wallet.save()
|
||||
|
||||
+17
-100
@@ -1,21 +1,10 @@
|
||||
from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey
|
||||
from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey
|
||||
from coincurve.utils import (
|
||||
pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi
|
||||
)
|
||||
from coincurve.ecdsa import CDATA_SIG_LENGTH
|
||||
from coincurve import PublicKey, PrivateKey as _PrivateKey
|
||||
|
||||
from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from .util import cachedproperty
|
||||
|
||||
|
||||
class KeyPath:
|
||||
RECEIVE = 0
|
||||
CHANGE = 1
|
||||
CHANNEL = 2
|
||||
|
||||
|
||||
class DerivationError(Exception):
|
||||
""" Raised when an invalid derivation occurs. """
|
||||
|
||||
@@ -82,30 +71,26 @@ class _KeyBase:
|
||||
return Base58.encode_check(self.extended_key())
|
||||
|
||||
|
||||
class PublicKey(_KeyBase):
|
||||
class PubKey(_KeyBase):
|
||||
""" A BIP32 public key. """
|
||||
|
||||
def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
|
||||
super().__init__(ledger, chain_code, n, depth, parent)
|
||||
if isinstance(pubkey, cPublicKey):
|
||||
if isinstance(pubkey, PublicKey):
|
||||
self.verifying_key = pubkey
|
||||
else:
|
||||
self.verifying_key = self._verifying_key_from_pubkey(pubkey)
|
||||
|
||||
@classmethod
|
||||
def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey':
|
||||
return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0)
|
||||
|
||||
@classmethod
|
||||
def _verifying_key_from_pubkey(cls, pubkey):
|
||||
""" Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """
|
||||
""" Converts a 33-byte compressed pubkey into an PublicKey object. """
|
||||
if not isinstance(pubkey, (bytes, bytearray)):
|
||||
raise TypeError('pubkey must be raw bytes')
|
||||
if len(pubkey) != 33:
|
||||
raise ValueError('pubkey must be 33 bytes')
|
||||
if pubkey[0] not in (2, 3):
|
||||
raise ValueError('invalid pubkey prefix byte')
|
||||
return cPublicKey(pubkey)
|
||||
return PublicKey(pubkey)
|
||||
|
||||
@cachedproperty
|
||||
def pubkey_bytes(self):
|
||||
@@ -120,7 +105,7 @@ class PublicKey(_KeyBase):
|
||||
def ec_point(self):
|
||||
return self.verifying_key.point()
|
||||
|
||||
def child(self, n: int) -> 'PublicKey':
|
||||
def child(self, n: int):
|
||||
""" Return the derived child extended pubkey at index N. """
|
||||
if not 0 <= n < (1 << 31):
|
||||
raise ValueError('invalid BIP32 public key child number')
|
||||
@@ -128,7 +113,7 @@ class PublicKey(_KeyBase):
|
||||
msg = self.pubkey_bytes + n.to_bytes(4, 'big')
|
||||
L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name
|
||||
derived_key = self.verifying_key.add(L_b)
|
||||
return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
|
||||
return PubKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
|
||||
|
||||
def identifier(self):
|
||||
""" Return the key's identifier as 20 bytes. """
|
||||
@@ -141,36 +126,6 @@ class PublicKey(_KeyBase):
|
||||
self.pubkey_bytes
|
||||
)
|
||||
|
||||
def verify(self, signature, digest) -> bool:
|
||||
""" Verify that a signature is valid for a 32 byte digest. """
|
||||
|
||||
if len(signature) != 64:
|
||||
raise ValueError('Signature must be 64 bytes long.')
|
||||
|
||||
if len(digest) != 32:
|
||||
raise ValueError('Digest must be 32 bytes long.')
|
||||
|
||||
key = self.verifying_key
|
||||
|
||||
raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
|
||||
|
||||
parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact(
|
||||
key.context.ctx, raw_signature, signature
|
||||
)
|
||||
assert parsed == 1
|
||||
|
||||
normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
|
||||
|
||||
libsecp256k1.secp256k1_ecdsa_signature_normalize(
|
||||
key.context.ctx, normalized_signature, raw_signature
|
||||
)
|
||||
|
||||
verified = libsecp256k1.secp256k1_ecdsa_verify(
|
||||
key.context.ctx, normalized_signature, digest, key.public_key
|
||||
)
|
||||
|
||||
return bool(verified)
|
||||
|
||||
|
||||
class PrivateKey(_KeyBase):
|
||||
"""A BIP32 private key."""
|
||||
@@ -179,7 +134,7 @@ class PrivateKey(_KeyBase):
|
||||
|
||||
def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
|
||||
super().__init__(ledger, chain_code, n, depth, parent)
|
||||
if isinstance(privkey, cPrivateKey):
|
||||
if isinstance(privkey, _PrivateKey):
|
||||
self.signing_key = privkey
|
||||
else:
|
||||
self.signing_key = self._signing_key_from_privkey(privkey)
|
||||
@@ -187,7 +142,7 @@ class PrivateKey(_KeyBase):
|
||||
@classmethod
|
||||
def _signing_key_from_privkey(cls, private_key):
|
||||
""" Converts a 32-byte private key into an coincurve.PrivateKey object. """
|
||||
return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
|
||||
return _PrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
|
||||
|
||||
@classmethod
|
||||
def _private_key_secret_exponent(cls, private_key):
|
||||
@@ -199,36 +154,24 @@ class PrivateKey(_KeyBase):
|
||||
return int.from_bytes(private_key, 'big')
|
||||
|
||||
@classmethod
|
||||
def from_seed(cls, ledger, seed) -> 'PrivateKey':
|
||||
def from_seed(cls, ledger, seed):
|
||||
# This hard-coded message string seems to be coin-independent...
|
||||
hmac = hmac_sha512(b'Bitcoin seed', seed)
|
||||
privkey, chain_code = hmac[:32], hmac[32:]
|
||||
return cls(ledger, privkey, chain_code, 0, 0)
|
||||
|
||||
@classmethod
|
||||
def from_pem(cls, ledger, pem) -> 'PrivateKey':
|
||||
der = pem_to_der(pem.encode())
|
||||
try:
|
||||
key_int = ECPrivateKey.load(der).native['private_key']
|
||||
except ValueError:
|
||||
key_int = PrivateKeyInfo.load(der).native['private_key']['private_key']
|
||||
private_key = cPrivateKey.from_int(key_int)
|
||||
return cls(ledger, private_key, bytes((0,)*32), 0, 0)
|
||||
|
||||
@cachedproperty
|
||||
def private_key_bytes(self):
|
||||
""" Return the serialized private key (no leading zero byte). """
|
||||
return self.signing_key.secret
|
||||
|
||||
@cachedproperty
|
||||
def public_key(self) -> PublicKey:
|
||||
def public_key(self):
|
||||
""" Return the corresponding extended public key. """
|
||||
verifying_key = self.signing_key.public_key
|
||||
parent_pubkey = self.parent.public_key if self.parent else None
|
||||
return PublicKey(
|
||||
self.ledger, verifying_key, self.chain_code,
|
||||
self.n, self.depth, parent_pubkey
|
||||
)
|
||||
return PubKey(self.ledger, verifying_key, self.chain_code, self.n, self.depth,
|
||||
parent_pubkey)
|
||||
|
||||
def ec_point(self):
|
||||
return self.public_key.ec_point()
|
||||
@@ -241,12 +184,11 @@ class PrivateKey(_KeyBase):
|
||||
""" Return the private key encoded in Wallet Import Format. """
|
||||
return self.ledger.private_key_to_wif(self.private_key_bytes)
|
||||
|
||||
@property
|
||||
def address(self):
|
||||
""" The public key as a P2PKH address. """
|
||||
return self.public_key.address
|
||||
|
||||
def child(self, n) -> 'PrivateKey':
|
||||
def child(self, n):
|
||||
""" Return the derived child extended private key at index N."""
|
||||
if not 0 <= n < (1 << 32):
|
||||
raise ValueError('invalid BIP32 private key child number')
|
||||
@@ -265,28 +207,6 @@ class PrivateKey(_KeyBase):
|
||||
""" Produce a signature for piece of data by double hashing it and signing the hash. """
|
||||
return self.signing_key.sign(data, hasher=double_sha256)
|
||||
|
||||
def sign_compact(self, digest):
|
||||
""" Produce a compact signature. """
|
||||
key = self.signing_key
|
||||
|
||||
signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
|
||||
signed = libsecp256k1.secp256k1_ecdsa_sign(
|
||||
key.context.ctx, signature, digest, key.secret,
|
||||
libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL
|
||||
)
|
||||
|
||||
if not signed:
|
||||
raise ValueError('The private key was invalid.')
|
||||
|
||||
serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH)
|
||||
compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact(
|
||||
key.context.ctx, serialized, signature
|
||||
)
|
||||
if compacted != 1:
|
||||
raise ValueError('The signature could not be compacted.')
|
||||
|
||||
return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH))
|
||||
|
||||
def identifier(self):
|
||||
"""Return the key's identifier as 20 bytes."""
|
||||
return self.public_key.identifier()
|
||||
@@ -298,12 +218,9 @@ class PrivateKey(_KeyBase):
|
||||
b'\0' + self.private_key_bytes
|
||||
)
|
||||
|
||||
def to_pem(self):
|
||||
return self.signing_key.to_pem()
|
||||
|
||||
|
||||
def _from_extended_key(ledger, ekey):
|
||||
"""Return a PublicKey or PrivateKey from an extended key raw bytes."""
|
||||
"""Return a PubKey or PrivateKey from an extended key raw bytes."""
|
||||
if not isinstance(ekey, (bytes, bytearray)):
|
||||
raise TypeError('extended key must be raw bytes')
|
||||
if len(ekey) != 78:
|
||||
@@ -315,7 +232,7 @@ def _from_extended_key(ledger, ekey):
|
||||
|
||||
if ekey[:4] == ledger.extended_public_key_prefix:
|
||||
pubkey = ekey[45:]
|
||||
key = PublicKey(ledger, pubkey, chain_code, n, depth)
|
||||
key = PubKey(ledger, pubkey, chain_code, n, depth)
|
||||
elif ekey[:4] == ledger.extended_private_key_prefix:
|
||||
if ekey[45] != 0:
|
||||
raise ValueError('invalid extended private key prefix byte')
|
||||
@@ -333,6 +250,6 @@ def from_extended_key_string(ledger, ekey_str):
|
||||
xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
|
||||
3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
|
||||
|
||||
return a PublicKey or PrivateKey.
|
||||
return a PubKey or PrivateKey.
|
||||
"""
|
||||
return _from_extended_key(ledger, Base58.decode_check(ekey_str))
|
||||
@@ -2,6 +2,7 @@ NULL_HASH32 = b'\x00'*32
|
||||
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
DUST = 1000
|
||||
|
||||
TIMEOUT = 30.0
|
||||
|
||||
|
||||
+3
-18
@@ -9,11 +9,10 @@ from dataclasses import dataclass
|
||||
from contextvars import ContextVar
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
|
||||
from datetime import date
|
||||
|
||||
from prometheus_client import Gauge, Counter, Histogram
|
||||
from lbry.utils import LockWithMetrics
|
||||
|
||||
from .bip32 import PublicKey
|
||||
from .bip32 import PubKey
|
||||
from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Input
|
||||
from .constants import TXO_TYPES, CLAIM_TYPES
|
||||
from .util import date_to_julian_day
|
||||
@@ -976,9 +975,7 @@ class Database(SQLiteMixin):
|
||||
sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)")
|
||||
return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only)
|
||||
|
||||
async def get_txos(
|
||||
self, wallet=None, no_tx=False, no_channel_info=False, read_only=False, **constraints
|
||||
) -> List[Output]:
|
||||
async def get_txos(self, wallet=None, no_tx=False, no_channel_info=False, read_only=False, **constraints):
|
||||
include_is_spent = constraints.get('include_is_spent', False)
|
||||
include_is_my_input = constraints.get('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
@@ -1204,7 +1201,7 @@ class Database(SQLiteMixin):
|
||||
addresses = await self.select_addresses(', '.join(cols), read_only=read_only, **constraints)
|
||||
if 'pubkey' in cols:
|
||||
for address in addresses:
|
||||
address['pubkey'] = PublicKey(
|
||||
address['pubkey'] = PubKey(
|
||||
self.ledger, address.pop('pubkey'), address.pop('chain_code'),
|
||||
address.pop('n'), address.pop('depth')
|
||||
)
|
||||
@@ -1244,18 +1241,6 @@ class Database(SQLiteMixin):
|
||||
async def set_address_history(self, address, history):
|
||||
await self._set_address_history(address, history)
|
||||
|
||||
async def is_channel_key_used(self, account, key: PublicKey):
|
||||
channels = await self.get_txos(
|
||||
accounts=[account], txo_type=TXO_TYPES['channel'],
|
||||
no_tx=True, no_channel_info=True
|
||||
)
|
||||
other_key_bytes = key.pubkey_bytes
|
||||
for channel in channels:
|
||||
claim = channel.can_decode_claim
|
||||
if claim and claim.channel.public_key_bytes == other_key_bytes:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def constrain_purchases(constraints):
|
||||
accounts = constraints.pop('accounts', None)
|
||||
|
||||
+2
-14
@@ -26,7 +26,7 @@ from .transaction import Transaction, Output
|
||||
from .header import Headers, UnvalidatedHeaders
|
||||
from .checkpoints import HASHES
|
||||
from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
|
||||
from .bip32 import PublicKey, PrivateKey
|
||||
from .bip32 import PubKey, PrivateKey
|
||||
from .coinselection import CoinSelector
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -226,7 +226,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return account.get_private_key(address_info['chain'], address_info['pubkey'].n)
|
||||
return None
|
||||
|
||||
async def get_public_key_for_address(self, wallet, address) -> Optional[PublicKey]:
|
||||
async def get_public_key_for_address(self, wallet, address) -> Optional[PubKey]:
|
||||
match = await self._get_account_and_address_info_for_address(wallet, address)
|
||||
if match:
|
||||
_, address_info = match
|
||||
@@ -365,10 +365,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
await self.db.close()
|
||||
await self.headers.close()
|
||||
|
||||
async def tasks_are_done(self):
|
||||
await self._update_tasks.done.wait()
|
||||
await self._other_tasks.done.wait()
|
||||
|
||||
@property
|
||||
def local_height_including_downloaded_height(self):
|
||||
return max(self.headers.height, self._download_height)
|
||||
@@ -474,7 +470,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
for address_manager in account.address_managers.values():
|
||||
await self.subscribe_addresses(address_manager, await address_manager.get_addresses())
|
||||
await account.ensure_address_gap()
|
||||
await account.deterministic_channel_keys.ensure_cache_primed()
|
||||
|
||||
async def unsubscribe_account(self, account: Account):
|
||||
for address in await account.get_addresses():
|
||||
@@ -555,7 +550,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
)
|
||||
remote_history_txids = {txid for txid, _ in remote_history}
|
||||
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
|
||||
self.maybe_has_channel_key(tx)
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
if len(pending_synced_history) % 100 == 0:
|
||||
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
@@ -623,12 +617,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
return tx
|
||||
|
||||
def maybe_has_channel_key(self, tx):
|
||||
for txo in tx._outputs:
|
||||
if txo.can_decode_claim and txo.claim.is_channel:
|
||||
for account in self.accounts:
|
||||
account.deterministic_channel_keys.maybe_generate_deterministic_key_for_channel(txo)
|
||||
|
||||
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False):
|
||||
batches = [[]]
|
||||
remote_heights = {}
|
||||
|
||||
@@ -5,9 +5,7 @@ import aiohttp
|
||||
|
||||
from lbry import wallet
|
||||
from lbry.wallet.orchstr8.node import (
|
||||
Conductor,
|
||||
get_lbcd_node_from_ledger,
|
||||
get_lbcwallet_node_from_ledger
|
||||
Conductor, get_blockchain_node_from_ledger
|
||||
)
|
||||
from lbry.wallet.orchstr8.service import ConductorService
|
||||
|
||||
@@ -18,11 +16,10 @@ def get_argument_parser():
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest='command', help='sub-command help')
|
||||
|
||||
subparsers.add_parser("download", help="Download lbcd and lbcwallet node binaries.")
|
||||
subparsers.add_parser("download", help="Download blockchain node binary.")
|
||||
|
||||
start = subparsers.add_parser("start", help="Start orchstr8 service.")
|
||||
start.add_argument("--lbcd", help="Hostname to start lbcd node.")
|
||||
start.add_argument("--lbcwallet", help="Hostname to start lbcwallet node.")
|
||||
start.add_argument("--blockchain", help="Hostname to start blockchain node.")
|
||||
start.add_argument("--spv", help="Hostname to start SPV server.")
|
||||
start.add_argument("--wallet", help="Hostname to start wallet daemon.")
|
||||
|
||||
@@ -50,8 +47,7 @@ def main():
|
||||
|
||||
if command == 'download':
|
||||
logging.getLogger('blockchain').setLevel(logging.INFO)
|
||||
get_lbcd_node_from_ledger(wallet).ensure()
|
||||
get_lbcwallet_node_from_ledger(wallet).ensure()
|
||||
get_blockchain_node_from_ledger(wallet).ensure()
|
||||
|
||||
elif command == 'generate':
|
||||
loop.run_until_complete(run_remote_command(
|
||||
@@ -61,12 +57,9 @@ def main():
|
||||
elif command == 'start':
|
||||
|
||||
conductor = Conductor()
|
||||
if getattr(args, 'lbcd', False):
|
||||
conductor.lbcd_node.hostname = args.lbcd
|
||||
loop.run_until_complete(conductor.start_lbcd())
|
||||
if getattr(args, 'lbcwallet', False):
|
||||
conductor.lbcwallet_node.hostname = args.lbcwallet
|
||||
loop.run_until_complete(conductor.start_lbcwallet())
|
||||
if getattr(args, 'blockchain', False):
|
||||
conductor.blockchain_node.hostname = args.blockchain
|
||||
loop.run_until_complete(conductor.start_blockchain())
|
||||
if getattr(args, 'spv', False):
|
||||
conductor.spv_node.hostname = args.spv
|
||||
loop.run_until_complete(conductor.start_spv())
|
||||
|
||||
+77
-269
@@ -8,7 +8,6 @@ import logging
|
||||
import tempfile
|
||||
import subprocess
|
||||
import importlib
|
||||
import platform
|
||||
from distutils.util import strtobool
|
||||
|
||||
from binascii import hexlify
|
||||
@@ -32,18 +31,11 @@ def get_spvserver_from_ledger(ledger_module):
|
||||
return getattr(spvserver_module, regtest_class_name)
|
||||
|
||||
|
||||
def get_lbcd_node_from_ledger(ledger_module):
|
||||
return LBCDNode(
|
||||
ledger_module.__lbcd_url__,
|
||||
ledger_module.__lbcd__,
|
||||
ledger_module.__lbcctl__
|
||||
)
|
||||
|
||||
def get_lbcwallet_node_from_ledger(ledger_module):
|
||||
return LBCWalletNode(
|
||||
ledger_module.__lbcwallet_url__,
|
||||
ledger_module.__lbcwallet__,
|
||||
ledger_module.__lbcctl__
|
||||
def get_blockchain_node_from_ledger(ledger_module):
|
||||
return BlockchainNode(
|
||||
ledger_module.__node_url__,
|
||||
os.path.join(ledger_module.__node_bin__, ledger_module.__node_daemon__),
|
||||
os.path.join(ledger_module.__node_bin__, ledger_module.__node_cli__)
|
||||
)
|
||||
|
||||
|
||||
@@ -53,52 +45,51 @@ class Conductor:
|
||||
self.manager_module = WalletManager
|
||||
self.spv_module = get_spvserver_from_ledger(lbry.wallet)
|
||||
|
||||
self.lbcd_node = get_lbcd_node_from_ledger(lbry.wallet)
|
||||
self.lbcwallet_node = get_lbcwallet_node_from_ledger(lbry.wallet)
|
||||
self.blockchain_node = get_blockchain_node_from_ledger(lbry.wallet)
|
||||
self.spv_node = SPVNode(self.spv_module)
|
||||
self.wallet_node = WalletNode(
|
||||
self.manager_module, RegTestLedger, default_seed=seed
|
||||
)
|
||||
self.hub_node = HubNode(__hub_url__, "hub", self.spv_node)
|
||||
|
||||
self.lbcd_started = False
|
||||
self.lbcwallet_started = False
|
||||
self.blockchain_started = False
|
||||
self.spv_started = False
|
||||
self.wallet_started = False
|
||||
self.hub_started = False
|
||||
|
||||
self.log = log.getChild('conductor')
|
||||
|
||||
async def start_lbcd(self):
|
||||
if not self.lbcd_started:
|
||||
asyncio.create_task(self.lbcd_node.start())
|
||||
await self.lbcd_node.running.wait()
|
||||
self.lbcd_started = True
|
||||
async def start_blockchain(self):
|
||||
if not self.blockchain_started:
|
||||
asyncio.create_task(self.blockchain_node.start())
|
||||
await self.blockchain_node.running.wait()
|
||||
await self.blockchain_node.generate(200)
|
||||
self.blockchain_started = True
|
||||
|
||||
async def stop_lbcd(self, cleanup=True):
|
||||
if self.lbcd_started:
|
||||
await self.lbcd_node.stop(cleanup)
|
||||
self.lbcd_started = False
|
||||
async def stop_blockchain(self):
|
||||
if self.blockchain_started:
|
||||
await self.blockchain_node.stop(cleanup=True)
|
||||
self.blockchain_started = False
|
||||
|
||||
async def start_hub(self):
|
||||
if not self.hub_started:
|
||||
asyncio.create_task(self.hub_node.start())
|
||||
await self.lbcwallet_node.running.wait()
|
||||
await self.blockchain_node.running.wait()
|
||||
self.hub_started = True
|
||||
|
||||
async def stop_hub(self, cleanup=True):
|
||||
async def stop_hub(self):
|
||||
if self.hub_started:
|
||||
await self.hub_node.stop(cleanup)
|
||||
await self.hub_node.stop(cleanup=True)
|
||||
self.hub_started = False
|
||||
|
||||
async def start_spv(self):
|
||||
if not self.spv_started:
|
||||
await self.spv_node.start(self.lbcwallet_node)
|
||||
await self.spv_node.start(self.blockchain_node)
|
||||
self.spv_started = True
|
||||
|
||||
async def stop_spv(self, cleanup=True):
|
||||
async def stop_spv(self):
|
||||
if self.spv_started:
|
||||
await self.spv_node.stop(cleanup)
|
||||
await self.spv_node.stop(cleanup=True)
|
||||
self.spv_started = False
|
||||
|
||||
async def start_wallet(self):
|
||||
@@ -106,31 +97,13 @@ class Conductor:
|
||||
await self.wallet_node.start(self.spv_node)
|
||||
self.wallet_started = True
|
||||
|
||||
async def stop_wallet(self, cleanup=True):
|
||||
async def stop_wallet(self):
|
||||
if self.wallet_started:
|
||||
await self.wallet_node.stop(cleanup)
|
||||
await self.wallet_node.stop(cleanup=True)
|
||||
self.wallet_started = False
|
||||
|
||||
async def start_lbcwallet(self, clean=True):
|
||||
if not self.lbcwallet_started:
|
||||
asyncio.create_task(self.lbcwallet_node.start())
|
||||
await self.lbcwallet_node.running.wait()
|
||||
if clean:
|
||||
mining_addr = await self.lbcwallet_node.get_new_address()
|
||||
self.lbcwallet_node.mining_addr = mining_addr
|
||||
await self.lbcwallet_node.generate(200)
|
||||
# unlock the wallet for the next 1 hour
|
||||
await self.lbcwallet_node.wallet_passphrase("password", 3600)
|
||||
self.lbcwallet_started = True
|
||||
|
||||
async def stop_lbcwallet(self, cleanup=True):
|
||||
if self.lbcwallet_started:
|
||||
await self.lbcwallet_node.stop(cleanup)
|
||||
self.lbcwallet_started = False
|
||||
|
||||
async def start(self):
|
||||
await self.start_lbcd()
|
||||
await self.start_lbcwallet()
|
||||
await self.start_blockchain()
|
||||
await self.start_spv()
|
||||
await self.start_wallet()
|
||||
|
||||
@@ -138,8 +111,7 @@ class Conductor:
|
||||
all_the_stops = [
|
||||
self.stop_wallet,
|
||||
self.stop_spv,
|
||||
self.stop_lbcwallet,
|
||||
self.stop_lbcd
|
||||
self.stop_blockchain
|
||||
]
|
||||
for stop in all_the_stops:
|
||||
try:
|
||||
@@ -147,11 +119,6 @@ class Conductor:
|
||||
except Exception as e:
|
||||
log.exception('Exception raised while stopping services:', exc_info=e)
|
||||
|
||||
async def clear_mempool(self):
|
||||
await self.stop_lbcwallet(cleanup=False)
|
||||
await self.stop_lbcd(cleanup=False)
|
||||
await self.start_lbcd()
|
||||
await self.start_lbcwallet(clean=False)
|
||||
|
||||
class WalletNode:
|
||||
|
||||
@@ -172,11 +139,10 @@ class WalletNode:
|
||||
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True, config=None):
|
||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||
os.mkdir(wallets_dir)
|
||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||
if not os.path.isdir(wallets_dir):
|
||||
os.mkdir(wallets_dir)
|
||||
with open(wallet_file_name, 'w') as wallet_file:
|
||||
wallet_file.write('{"version": 1, "accounts": []}\n')
|
||||
with open(wallet_file_name, 'w') as wallet_file:
|
||||
wallet_file.write('{"version": 1, "accounts": []}\n')
|
||||
self.manager = self.manager_class.from_config({
|
||||
'ledgers': {
|
||||
self.ledger_class.get_id(): {
|
||||
@@ -232,14 +198,14 @@ class SPVNode:
|
||||
self.stopped = False
|
||||
self.index_name = uuid4().hex
|
||||
|
||||
async def start(self, lbcwallet_node: 'LBCWalletNode', extraconf=None):
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
conf = {
|
||||
'DESCRIPTION': '',
|
||||
'PAYMENT_ADDRESS': '',
|
||||
'DAILY_FEE': '0',
|
||||
'DB_DIRECTORY': self.data_path,
|
||||
'DAEMON_URL': lbcwallet_node.rpc_url,
|
||||
'DAEMON_URL': blockchain_node.rpc_url,
|
||||
'REORG_LIMIT': '100',
|
||||
'HOST': self.hostname,
|
||||
'TCP_PORT': str(self.port),
|
||||
@@ -274,19 +240,18 @@ class SPVNode:
|
||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
|
||||
|
||||
class LBCDProcess(asyncio.SubprocessProtocol):
|
||||
class BlockchainProcess(asyncio.SubprocessProtocol):
|
||||
|
||||
IGNORE_OUTPUT = [
|
||||
b'keypool keep',
|
||||
b'keypool reserve',
|
||||
b'keypool return',
|
||||
b'Block submitted',
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
self.ready = asyncio.Event()
|
||||
self.stopped = asyncio.Event()
|
||||
self.log = log.getChild('lbcd')
|
||||
self.log = log.getChild('blockchain')
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
if self.log and not any(ignore in data for ignore in self.IGNORE_OUTPUT):
|
||||
@@ -297,7 +262,7 @@ class LBCDProcess(asyncio.SubprocessProtocol):
|
||||
if b'Error:' in data:
|
||||
self.ready.set()
|
||||
raise SystemError(data.decode())
|
||||
if b'RPCS: RPC server listening on' in data:
|
||||
if b'Done loading' in data:
|
||||
self.ready.set()
|
||||
|
||||
def process_exited(self):
|
||||
@@ -305,34 +270,10 @@ class LBCDProcess(asyncio.SubprocessProtocol):
|
||||
self.ready.set()
|
||||
|
||||
|
||||
class WalletProcess(asyncio.SubprocessProtocol):
|
||||
class BlockchainNode:
|
||||
|
||||
IGNORE_OUTPUT = [
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
self.ready = asyncio.Event()
|
||||
self.stopped = asyncio.Event()
|
||||
self.log = log.getChild('lbcwallet')
|
||||
self.transport: Optional[asyncio.transports.SubprocessTransport] = None
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
if self.log and not any(ignore in data for ignore in self.IGNORE_OUTPUT):
|
||||
if b'Error:' in data:
|
||||
self.log.error(data.decode())
|
||||
else:
|
||||
self.log.info(data.decode())
|
||||
if b'Error:' in data:
|
||||
self.ready.set()
|
||||
raise SystemError(data.decode())
|
||||
if b'WLLT: Finished rescan' in data:
|
||||
self.ready.set()
|
||||
|
||||
def process_exited(self):
|
||||
self.stopped.set()
|
||||
self.ready.set()
|
||||
|
||||
class LBCDNode:
|
||||
P2SH_SEGWIT_ADDRESS = "p2sh-segwit"
|
||||
BECH32_ADDRESS = "bech32"
|
||||
|
||||
def __init__(self, url, daemon, cli):
|
||||
self.latest_release_url = url
|
||||
@@ -340,22 +281,28 @@ class LBCDNode:
|
||||
self.bin_dir = os.path.join(self.project_dir, 'bin')
|
||||
self.daemon_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.cli_bin = os.path.join(self.bin_dir, cli)
|
||||
self.log = log.getChild('lbcd')
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
self.log = log.getChild('blockchain')
|
||||
self.data_path = None
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.block_expected = 0
|
||||
self.hostname = 'localhost'
|
||||
self.peerport = 29246
|
||||
self.rpcport = 29245
|
||||
self.peerport = 9246 + 2 # avoid conflict with default peer port
|
||||
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
|
||||
self.rpcuser = 'rpcuser'
|
||||
self.rpcpassword = 'rpcpassword'
|
||||
self.stopped = True
|
||||
self.stopped = False
|
||||
self.restart_ready = asyncio.Event()
|
||||
self.restart_ready.set()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
def rpc_url(self):
|
||||
return f'http://{self.rpcuser}:{self.rpcpassword}@{self.hostname}:{self.rpcport}/'
|
||||
|
||||
def is_expected_block(self, e: BlockHeightEvent):
|
||||
return self.block_expected == e.height
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
@@ -364,12 +311,6 @@ class LBCDNode:
|
||||
)
|
||||
|
||||
def download(self):
|
||||
uname = platform.uname()
|
||||
target_os = str.lower(uname.system)
|
||||
target_arch = str.replace(uname.machine, 'x86_64', 'amd64')
|
||||
target_platform = target_os + '_' + target_arch
|
||||
self.latest_release_url = str.replace(self.latest_release_url, 'TARGET_PLATFORM', target_platform)
|
||||
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir,
|
||||
self.latest_release_url[self.latest_release_url.rfind('/')+1:]
|
||||
@@ -404,34 +345,34 @@ class LBCDNode:
|
||||
|
||||
async def start(self):
|
||||
assert self.ensure()
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.get_child_watcher().attach_loop(loop)
|
||||
command = [
|
||||
self.daemon_bin,
|
||||
'--notls',
|
||||
f'--datadir={self.data_path}',
|
||||
'--regtest', f'--listen=127.0.0.1:{self.peerport}', f'--rpclisten=127.0.0.1:{self.rpcport}',
|
||||
'--txindex', f'--rpcuser={self.rpcuser}', f'--rpcpass={self.rpcpassword}'
|
||||
f'-datadir={self.data_path}', '-printtoconsole', '-regtest', '-server', '-txindex',
|
||||
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}',
|
||||
f'-port={self.peerport}'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
while self.stopped:
|
||||
while not self.stopped:
|
||||
if self.running.is_set():
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
await self.restart_ready.wait()
|
||||
try:
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
LBCDProcess, *command
|
||||
BlockchainProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
self.running.set()
|
||||
self.stopped = False
|
||||
except asyncio.CancelledError:
|
||||
self.running.clear()
|
||||
raise
|
||||
except Exception as e:
|
||||
self.running.clear()
|
||||
log.exception('failed to start lbcd', exc_info=e)
|
||||
log.exception('failed to start lbrycrdd', exc_info=e)
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
self.stopped = True
|
||||
@@ -440,161 +381,35 @@ class LBCDNode:
|
||||
await self.protocol.stopped.wait()
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.log.info("Done shutting down " + self.daemon_bin)
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
self.running.clear()
|
||||
|
||||
async def clear_mempool(self):
|
||||
self.restart_ready.clear()
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
self.transport.close()
|
||||
self.running.clear()
|
||||
os.remove(os.path.join(self.data_path, 'regtest', 'mempool.dat'))
|
||||
self.restart_ready.set()
|
||||
await self.running.wait()
|
||||
|
||||
def cleanup(self):
|
||||
assert self.stopped
|
||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
|
||||
|
||||
class LBCWalletNode:
|
||||
P2SH_SEGWIT_ADDRESS = "p2sh-segwit"
|
||||
BECH32_ADDRESS = "bech32"
|
||||
|
||||
def __init__(self, url, lbcwallet, cli):
|
||||
self.latest_release_url = url
|
||||
self.project_dir = os.path.dirname(os.path.dirname(__file__))
|
||||
self.bin_dir = os.path.join(self.project_dir, 'bin')
|
||||
self.lbcwallet_bin = os.path.join(self.bin_dir, lbcwallet)
|
||||
self.cli_bin = os.path.join(self.bin_dir, cli)
|
||||
self.log = log.getChild('lbcwallet')
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.hostname = 'localhost'
|
||||
self.lbcd_rpcport = 29245
|
||||
self.lbcwallet_rpcport = 29244
|
||||
self.rpcuser = 'rpcuser'
|
||||
self.rpcpassword = 'rpcpassword'
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
self.stopped = True
|
||||
self.running = asyncio.Event()
|
||||
self.block_expected = 0
|
||||
self.mining_addr = ''
|
||||
|
||||
@property
|
||||
def rpc_url(self):
|
||||
# FIXME: somehow the hub/sdk doesn't learn the blocks through the Walet RPC port, why?
|
||||
# return f'http://{self.rpcuser}:{self.rpcpassword}@{self.hostname}:{self.lbcwallet_rpcport}/'
|
||||
return f'http://{self.rpcuser}:{self.rpcpassword}@{self.hostname}:{self.lbcd_rpcport}/'
|
||||
|
||||
def is_expected_block(self, e: BlockHeightEvent):
|
||||
return self.block_expected == e.height
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
os.path.exists(self.lbcwallet_bin)
|
||||
)
|
||||
|
||||
def download(self):
|
||||
uname = platform.uname()
|
||||
target_os = str.lower(uname.system)
|
||||
target_arch = str.replace(uname.machine, 'x86_64', 'amd64')
|
||||
target_platform = target_os + '_' + target_arch
|
||||
self.latest_release_url = str.replace(self.latest_release_url, 'TARGET_PLATFORM', target_platform)
|
||||
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir,
|
||||
self.latest_release_url[self.latest_release_url.rfind('/')+1:]
|
||||
)
|
||||
|
||||
if not os.path.exists(self.bin_dir):
|
||||
os.mkdir(self.bin_dir)
|
||||
|
||||
if not os.path.exists(downloaded_file):
|
||||
self.log.info('Downloading: %s', self.latest_release_url)
|
||||
with urllib.request.urlopen(self.latest_release_url) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
shutil.copyfileobj(response, out_file)
|
||||
|
||||
self.log.info('Extracting: %s', downloaded_file)
|
||||
|
||||
if downloaded_file.endswith('.zip'):
|
||||
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||
dotzip.extractall(self.bin_dir)
|
||||
# zipfile bug https://bugs.python.org/issue15795
|
||||
os.chmod(self.lbcwallet_bin, 0o755)
|
||||
|
||||
elif downloaded_file.endswith('.tar.gz'):
|
||||
with tarfile.open(downloaded_file) as tar:
|
||||
tar.extractall(self.bin_dir)
|
||||
|
||||
return self.exists
|
||||
|
||||
def ensure(self):
|
||||
return self.exists or self.download()
|
||||
|
||||
async def start(self):
|
||||
assert self.ensure()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.get_child_watcher().attach_loop(loop)
|
||||
|
||||
command = [
|
||||
self.lbcwallet_bin,
|
||||
'--noservertls', '--noclienttls',
|
||||
'--regtest',
|
||||
f'--rpcconnect=127.0.0.1:{self.lbcd_rpcport}', f'--rpclisten=127.0.0.1:{self.lbcwallet_rpcport}',
|
||||
'--createtemp', f'--appdata={self.data_path}',
|
||||
f'--username={self.rpcuser}', f'--password={self.rpcpassword}'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
while self.stopped:
|
||||
if self.running.is_set():
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
try:
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
WalletProcess, *command
|
||||
)
|
||||
self.protocol.transport = self.transport
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
self.running.set()
|
||||
self.stopped = False
|
||||
except asyncio.CancelledError:
|
||||
self.running.clear()
|
||||
raise
|
||||
except Exception as e:
|
||||
self.running.clear()
|
||||
log.exception('failed to start lbcwallet', exc_info=e)
|
||||
|
||||
def cleanup(self):
|
||||
assert self.stopped
|
||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
self.stopped = True
|
||||
try:
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.log.info("Done shutting down " + self.lbcwallet_bin)
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
self.running.clear()
|
||||
|
||||
async def _cli_cmnd(self, *args):
|
||||
cmnd_args = [
|
||||
self.cli_bin,
|
||||
f'--rpcuser={self.rpcuser}', f'--rpcpass={self.rpcpassword}', '--notls', '--regtest', '--wallet'
|
||||
self.cli_bin, f'-datadir={self.data_path}', '-regtest',
|
||||
f'-rpcuser={self.rpcuser}', f'-rpcpassword={self.rpcpassword}', f'-rpcport={self.rpcport}'
|
||||
] + list(args)
|
||||
self.log.info(' '.join(cmnd_args))
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.get_child_watcher().attach_loop(loop)
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmnd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
*cmnd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
out, err = await process.communicate()
|
||||
out, _ = await process.communicate()
|
||||
result = out.decode().strip()
|
||||
err = err.decode().strip()
|
||||
if len(result) <= 0 and err.startswith('-'):
|
||||
raise Exception(err)
|
||||
if err and 'creating a default config file' not in err:
|
||||
log.warning(err)
|
||||
self.log.info(result)
|
||||
if result.startswith('error code'):
|
||||
raise Exception(result)
|
||||
@@ -602,14 +417,7 @@ class LBCWalletNode:
|
||||
|
||||
def generate(self, blocks):
|
||||
self.block_expected += blocks
|
||||
return self._cli_cmnd('generatetoaddress', str(blocks), self.mining_addr)
|
||||
|
||||
def generate_to_address(self, blocks, addr):
|
||||
self.block_expected += blocks
|
||||
return self._cli_cmnd('generatetoaddress', str(blocks), addr)
|
||||
|
||||
def wallet_passphrase(self, passphrase, timeout):
|
||||
return self._cli_cmnd('walletpassphrase', passphrase, str(timeout))
|
||||
return self._cli_cmnd('generate', str(blocks))
|
||||
|
||||
def invalidate_block(self, blockhash):
|
||||
return self._cli_cmnd('invalidateblock', blockhash)
|
||||
@@ -626,7 +434,7 @@ class LBCWalletNode:
|
||||
def get_raw_change_address(self):
|
||||
return self._cli_cmnd('getrawchangeaddress')
|
||||
|
||||
def get_new_address(self, address_type='legacy'):
|
||||
def get_new_address(self, address_type):
|
||||
return self._cli_cmnd('getnewaddress', "", address_type)
|
||||
|
||||
async def get_balance(self):
|
||||
@@ -642,10 +450,7 @@ class LBCWalletNode:
|
||||
return self._cli_cmnd('createrawtransaction', json.dumps(inputs), json.dumps(outputs))
|
||||
|
||||
async def sign_raw_transaction_with_wallet(self, tx):
|
||||
# the "withwallet" portion should only come into play if we are doing segwit.
|
||||
# and "withwallet" doesn't exist on lbcd yet.
|
||||
result = await self._cli_cmnd('signrawtransaction', tx)
|
||||
return json.loads(result)['hex'].encode()
|
||||
return json.loads(await self._cli_cmnd('signrawtransactionwithwallet', tx))['hex'].encode()
|
||||
|
||||
def decode_raw_transaction(self, tx):
|
||||
return self._cli_cmnd('decoderawtransaction', hexlify(tx.raw).decode())
|
||||
@@ -695,6 +500,8 @@ class HubNode:
|
||||
self.hostname = 'localhost'
|
||||
self.rpcport = 50051 # avoid conflict with default rpc port
|
||||
self.stopped = False
|
||||
self.restart_ready = asyncio.Event()
|
||||
self.restart_ready.set()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
@@ -751,6 +558,7 @@ class HubNode:
|
||||
if self.running.is_set():
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
await self.restart_ready.wait()
|
||||
try:
|
||||
if not self.debug:
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
|
||||
@@ -61,10 +61,8 @@ class ConductorService:
|
||||
#set_logging(
|
||||
# self.stack.ledger_module, logging.DEBUG, WebSocketLogHandler(self.send_message)
|
||||
#)
|
||||
self.stack.lbcd_started or await self.stack.start_lbcd()
|
||||
self.send_message({'type': 'service', 'name': 'lbcd', 'port': self.stack.lbcd_node.port})
|
||||
self.stack.lbcwallet_started or await self.stack.start_lbcwallet()
|
||||
self.send_message({'type': 'service', 'name': 'lbcwallet', 'port': self.stack.lbcwallet_node.port})
|
||||
self.stack.blockchain_started or await self.stack.start_blockchain()
|
||||
self.send_message({'type': 'service', 'name': 'blockchain', 'port': self.stack.blockchain_node.port})
|
||||
self.stack.spv_started or await self.stack.start_spv()
|
||||
self.send_message({'type': 'service', 'name': 'spv', 'port': self.stack.spv_node.port})
|
||||
self.stack.wallet_started or await self.stack.start_wallet()
|
||||
@@ -76,7 +74,7 @@ class ConductorService:
|
||||
async def generate(self, request):
|
||||
data = await request.post()
|
||||
blocks = data.get('blocks', 1)
|
||||
await self.stack.lbcwallet_node.generate(int(blocks))
|
||||
await self.stack.blockchain_node.generate(int(blocks))
|
||||
return json_response({'blocks': blocks})
|
||||
|
||||
async def transfer(self, request):
|
||||
@@ -87,14 +85,11 @@ class ConductorService:
|
||||
if not address:
|
||||
raise ValueError("No address was provided.")
|
||||
amount = data.get('amount', 1)
|
||||
txid = await self.stack.blockchain_node.send_to_address(address, amount)
|
||||
if self.stack.wallet_started:
|
||||
watcher = self.stack.wallet_node.ledger.on_transaction.where(
|
||||
lambda e: e.address == address # and e.tx.id == txid -- might stall; see send_to_address_and_wait
|
||||
await self.stack.wallet_node.ledger.on_transaction.where(
|
||||
lambda e: e.tx.id == txid and e.address == address
|
||||
)
|
||||
txid = await self.stack.lbcwallet_node.send_to_address(address, amount)
|
||||
await watcher
|
||||
else:
|
||||
txid = await self.stack.lbcwallet_node.send_to_address(address, amount)
|
||||
return json_response({
|
||||
'address': address,
|
||||
'amount': amount,
|
||||
@@ -103,7 +98,7 @@ class ConductorService:
|
||||
|
||||
async def balance(self, _):
|
||||
return json_response({
|
||||
'balance': await self.stack.lbcwallet_node.get_balance()
|
||||
'balance': await self.stack.blockchain_node.get_balance()
|
||||
})
|
||||
|
||||
async def log(self, request):
|
||||
@@ -134,7 +129,7 @@ class ConductorService:
|
||||
'type': 'status',
|
||||
'height': self.stack.wallet_node.ledger.headers.height,
|
||||
'balance': satoshis_to_coins(await self.stack.wallet_node.account.get_balance()),
|
||||
'miner': await self.stack.lbcwallet_node.get_balance()
|
||||
'miner': await self.stack.blockchain_node.get_balance()
|
||||
})
|
||||
|
||||
def send_message(self, msg):
|
||||
|
||||
@@ -507,7 +507,7 @@ class BlockProcessor:
|
||||
channel_pub_key_bytes = channel_meta.channel.public_key_bytes
|
||||
if channel_pub_key_bytes:
|
||||
channel_signature_is_valid = Output.is_signature_valid(
|
||||
txo.signable.signature, txo.get_signature_digest(self.ledger), channel_pub_key_bytes
|
||||
txo.get_encoded_signature(), txo.get_signature_digest(self.ledger), channel_pub_key_bytes
|
||||
)
|
||||
if channel_signature_is_valid:
|
||||
self.pending_channel_counts[signing_channel_hash] += 1
|
||||
|
||||
@@ -53,7 +53,7 @@ class Daemon:
|
||||
self.max_retry = max_retry
|
||||
self._height = None
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector(ssl=False)
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
self._block_hash_cache = LRUCacheWithMetrics(100000)
|
||||
self._block_cache = LRUCacheWithMetrics(2 ** 13, metric_name='block', namespace=NAMESPACE)
|
||||
|
||||
@@ -250,14 +250,14 @@ class Daemon:
|
||||
async def deserialised_block(self, hex_hash):
|
||||
"""Return the deserialised block with the given hex hash."""
|
||||
if hex_hash not in self._block_cache:
|
||||
block = await self._send_single('getblock', (hex_hash, 1))
|
||||
block = await self._send_single('getblock', (hex_hash, True))
|
||||
self._block_cache[hex_hash] = block
|
||||
return block
|
||||
return self._block_cache[hex_hash]
|
||||
|
||||
async def raw_blocks(self, hex_hashes):
|
||||
"""Return the raw binary blocks with the given hex hashes."""
|
||||
params_iterable = ((h, 0) for h in hex_hashes)
|
||||
params_iterable = ((h, False) for h in hex_hashes)
|
||||
blocks = await self._send_vector('getblock', params_iterable)
|
||||
# Convert hex string to bytes
|
||||
return [hex_to_bytes(block) for block in blocks]
|
||||
@@ -334,7 +334,42 @@ class LBCDaemon(Daemon):
|
||||
async def getrawtransaction(self, hex_hash, verbose=False):
|
||||
return await super().getrawtransaction(hex_hash=hex_hash, verbose=verbose)
|
||||
|
||||
@handles_errors
|
||||
async def getclaimbyid(self, claim_id):
|
||||
'''Given a claim id, retrieves claim information.'''
|
||||
return await self._send_single('getclaimbyid', (claim_id,))
|
||||
|
||||
@handles_errors
|
||||
async def getclaimsbyids(self, claim_ids):
|
||||
'''Given a list of claim ids, batches calls to retrieve claim information.'''
|
||||
return await self._send_vector('getclaimbyid', ((claim_id,) for claim_id in claim_ids))
|
||||
|
||||
@handles_errors
|
||||
async def getclaimsforname(self, name):
|
||||
'''Given a name, retrieves all claims matching that name.'''
|
||||
return await self._send_single('getclaimsforname', (name,))
|
||||
|
||||
@handles_errors
|
||||
async def getclaimsfortx(self, txid):
|
||||
'''Given a txid, returns the claims it make.'''
|
||||
return await self._send_single('getclaimsfortx', (txid,)) or []
|
||||
|
||||
@handles_errors
|
||||
async def getnameproof(self, name, block_hash=None):
|
||||
'''Given a name and optional block_hash, returns a name proof and winner, if any.'''
|
||||
return await self._send_single('getnameproof', (name, block_hash,) if block_hash else (name,))
|
||||
|
||||
@handles_errors
|
||||
async def getvalueforname(self, name):
|
||||
'''Given a name, returns the winning claim value.'''
|
||||
return await self._send_single('getvalueforname', (name,))
|
||||
|
||||
@handles_errors
|
||||
async def getnamesintrie(self):
|
||||
'''Given a name, returns the winning claim value.'''
|
||||
return await self._send_single('getnamesintrie')
|
||||
|
||||
@handles_errors
|
||||
async def claimname(self, name, hexvalue, amount):
|
||||
'''Claim a name, used for functional tests only.'''
|
||||
return await self._send_single('claimname', (name, hexvalue, float(amount)))
|
||||
+11
-112
@@ -1,130 +1,32 @@
|
||||
import struct
|
||||
import rocksdb
|
||||
from typing import Optional
|
||||
from lbry.wallet.server.db import DB_PREFIXES
|
||||
from lbry.wallet.server.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete
|
||||
|
||||
|
||||
class RocksDBStore:
|
||||
def __init__(self, path: str, cache_mb: int, max_open_files: int, secondary_path: str = ''):
|
||||
# Use snappy compression (the default)
|
||||
self.path = path
|
||||
self._max_open_files = max_open_files
|
||||
self.db = rocksdb.DB(path, self.get_options(), secondary_name=secondary_path)
|
||||
# self.multi_get = self.db.multi_get
|
||||
|
||||
def get_options(self):
|
||||
return rocksdb.Options(
|
||||
create_if_missing=True, use_fsync=True, target_file_size_base=33554432,
|
||||
max_open_files=self._max_open_files
|
||||
)
|
||||
|
||||
class KeyValueStorage:
|
||||
def get(self, key: bytes, fill_cache: bool = True) -> Optional[bytes]:
|
||||
return self.db.get(key, fill_cache=fill_cache)
|
||||
raise NotImplemented()
|
||||
|
||||
def iterator(self, reverse=False, start=None, stop=None, include_start=True, include_stop=False, prefix=None,
|
||||
include_key=True, include_value=True, fill_cache=True):
|
||||
return RocksDBIterator(
|
||||
self.db, reverse=reverse, start=start, stop=stop, include_start=include_start, include_stop=include_stop,
|
||||
prefix=prefix, include_key=include_key, include_value=include_value
|
||||
)
|
||||
raise NotImplemented()
|
||||
|
||||
def write_batch(self, disable_wal: bool = False, sync: bool = False):
|
||||
return RocksDBWriteBatch(self.db, sync=sync, disable_wal=disable_wal)
|
||||
def write_batch(self, transaction: bool = False):
|
||||
raise NotImplemented()
|
||||
|
||||
def close(self):
|
||||
self.db.close()
|
||||
self.db = None
|
||||
raise NotImplemented()
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self.db is None
|
||||
|
||||
def try_catch_up_with_primary(self):
|
||||
self.db.try_catch_up_with_primary()
|
||||
|
||||
|
||||
class RocksDBWriteBatch:
|
||||
def __init__(self, db: rocksdb.DB, sync: bool = False, disable_wal: bool = False):
|
||||
self.batch = rocksdb.WriteBatch()
|
||||
self.db = db
|
||||
self.sync = sync
|
||||
self.disable_wal = disable_wal
|
||||
|
||||
def __enter__(self):
|
||||
return self.batch
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if not exc_val:
|
||||
self.db.write(self.batch, sync=self.sync, disable_wal=self.disable_wal)
|
||||
|
||||
|
||||
class RocksDBIterator:
|
||||
"""An iterator for RocksDB."""
|
||||
|
||||
__slots__ = [
|
||||
'start',
|
||||
'prefix',
|
||||
'stop',
|
||||
'iterator',
|
||||
'include_key',
|
||||
'include_value',
|
||||
'prev_k',
|
||||
'reverse',
|
||||
'include_start',
|
||||
'include_stop'
|
||||
]
|
||||
|
||||
def __init__(self, db: rocksdb.DB, prefix: bytes = None, start: bool = None, stop: bytes = None,
|
||||
include_key: bool = True, include_value: bool = True, reverse: bool = False,
|
||||
include_start: bool = True, include_stop: bool = False):
|
||||
assert (start is None and stop is None) or (prefix is None), 'cannot use start/stop and prefix'
|
||||
self.start = start
|
||||
self.prefix = prefix
|
||||
self.stop = stop
|
||||
self.iterator = db.iteritems() if not reverse else reversed(db.iteritems())
|
||||
if prefix is not None:
|
||||
self.iterator.seek(prefix)
|
||||
elif start is not None:
|
||||
self.iterator.seek(start)
|
||||
self.include_key = include_key
|
||||
self.include_value = include_value
|
||||
self.prev_k = None
|
||||
self.reverse = reverse
|
||||
self.include_start = include_start
|
||||
self.include_stop = include_stop
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def _check_stop_iteration(self, key: bytes):
|
||||
if self.stop is not None and (key.startswith(self.stop) or self.stop < key[:len(self.stop)]):
|
||||
raise StopIteration
|
||||
elif self.start is not None and self.start > key[:len(self.start)]:
|
||||
raise StopIteration
|
||||
elif self.prefix is not None and not key.startswith(self.prefix):
|
||||
raise StopIteration
|
||||
|
||||
def __next__(self):
|
||||
# TODO: include start/stop on/off
|
||||
# check for needing to stop from previous iteration
|
||||
if self.prev_k is not None:
|
||||
self._check_stop_iteration(self.prev_k)
|
||||
k, v = next(self.iterator)
|
||||
self._check_stop_iteration(k)
|
||||
self.prev_k = k
|
||||
|
||||
if self.include_key and self.include_value:
|
||||
return k, v
|
||||
elif self.include_key:
|
||||
return k
|
||||
return v
|
||||
raise NotImplemented()
|
||||
|
||||
|
||||
class PrefixDB:
|
||||
UNDO_KEY_STRUCT = struct.Struct(b'>Q')
|
||||
|
||||
def __init__(self, db: RocksDBStore, max_undo_depth: int = 200, unsafe_prefixes=None):
|
||||
def __init__(self, db: KeyValueStorage, max_undo_depth: int = 200, unsafe_prefixes=None):
|
||||
self._db = db
|
||||
self._op_stack = RevertableOpStack(db.get, unsafe_prefixes=unsafe_prefixes)
|
||||
self._max_undo_depth = max_undo_depth
|
||||
@@ -135,7 +37,7 @@ class PrefixDB:
|
||||
Changes written cannot be undone
|
||||
"""
|
||||
try:
|
||||
with self._db.write_batch(sync=True) as batch:
|
||||
with self._db.write_batch(transaction=True) as batch:
|
||||
batch_put = batch.put
|
||||
batch_delete = batch.delete
|
||||
for staged_change in self._op_stack:
|
||||
@@ -159,7 +61,7 @@ class PrefixDB:
|
||||
include_value=False
|
||||
))
|
||||
try:
|
||||
with self._db.write_batch(sync=True) as batch:
|
||||
with self._db.write_batch(transaction=True) as batch:
|
||||
batch_put = batch.put
|
||||
batch_delete = batch.delete
|
||||
for staged_change in self._op_stack:
|
||||
@@ -180,7 +82,7 @@ class PrefixDB:
|
||||
undo_key = DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height)
|
||||
self._op_stack.apply_packed_undo_ops(self._db.get(undo_key))
|
||||
try:
|
||||
with self._db.write_batch(sync=True) as batch:
|
||||
with self._db.write_batch(transaction=True) as batch:
|
||||
batch_put = batch.put
|
||||
batch_delete = batch.delete
|
||||
for staged_change in self._op_stack:
|
||||
@@ -206,9 +108,6 @@ class PrefixDB:
|
||||
if not self._db.closed:
|
||||
self._db.close()
|
||||
|
||||
def try_catch_up_with_primary(self):
|
||||
self._db.try_catch_up_with_primary()
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
return self._db.closed
|
||||
|
||||
@@ -4,7 +4,7 @@ import array
|
||||
import base64
|
||||
from typing import Union, Tuple, NamedTuple, Optional
|
||||
from lbry.wallet.server.db import DB_PREFIXES
|
||||
from lbry.wallet.server.db.db import RocksDBStore, PrefixDB
|
||||
from lbry.wallet.server.db.db import KeyValueStorage, PrefixDB
|
||||
from lbry.wallet.server.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete
|
||||
from lbry.schema.url import normalize_name
|
||||
|
||||
@@ -38,7 +38,7 @@ class PrefixRow(metaclass=PrefixRowType):
|
||||
value_struct: struct.Struct
|
||||
key_part_lambdas = []
|
||||
|
||||
def __init__(self, db: RocksDBStore, op_stack: RevertableOpStack):
|
||||
def __init__(self, db: KeyValueStorage, op_stack: RevertableOpStack):
|
||||
self._db = db
|
||||
self._op_stack = op_stack
|
||||
|
||||
@@ -1595,10 +1595,40 @@ class BlockTxsPrefixRow(PrefixRow):
|
||||
return cls.pack_key(height), cls.pack_value(tx_hashes)
|
||||
|
||||
|
||||
class LevelDBStore(KeyValueStorage):
|
||||
def __init__(self, path: str, cache_mb: int, max_open_files: int):
|
||||
import plyvel
|
||||
self.db = plyvel.DB(
|
||||
path, create_if_missing=True, max_open_files=max_open_files,
|
||||
lru_cache_size=cache_mb * 1024 * 1024, write_buffer_size=64 * 1024 * 1024,
|
||||
max_file_size=1024 * 1024 * 64, bloom_filter_bits=32
|
||||
)
|
||||
|
||||
def get(self, key: bytes, fill_cache: bool = True) -> Optional[bytes]:
|
||||
return self.db.get(key, fill_cache=fill_cache)
|
||||
|
||||
def iterator(self, reverse=False, start=None, stop=None, include_start=True, include_stop=False, prefix=None,
|
||||
include_key=True, include_value=True, fill_cache=True):
|
||||
return self.db.iterator(
|
||||
reverse=reverse, start=start, stop=stop, include_start=include_start, include_stop=include_stop,
|
||||
prefix=prefix, include_key=include_key, include_value=include_value, fill_cache=fill_cache
|
||||
)
|
||||
|
||||
def write_batch(self, transaction: bool = False, sync: bool = False):
|
||||
return self.db.write_batch(transaction=transaction, sync=sync)
|
||||
|
||||
def close(self):
|
||||
return self.db.close()
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self.db.closed
|
||||
|
||||
|
||||
class HubDB(PrefixDB):
|
||||
def __init__(self, path: str, cache_mb: int = 128, reorg_limit: int = 200, max_open_files: int = 512,
|
||||
secondary_path: str = '', unsafe_prefixes: Optional[typing.Set[bytes]] = None):
|
||||
db = RocksDBStore(path, cache_mb, max_open_files, secondary_path=secondary_path)
|
||||
unsafe_prefixes: Optional[typing.Set[bytes]] = None):
|
||||
db = LevelDBStore(path, cache_mb, max_open_files)
|
||||
super().__init__(db, reorg_limit, unsafe_prefixes=unsafe_prefixes)
|
||||
self.claim_to_support = ClaimToSupportPrefixRow(db, self._op_stack)
|
||||
self.support_to_claim = SupportToClaimPrefixRow(db, self._op_stack)
|
||||
|
||||
@@ -26,10 +26,7 @@ class Server:
|
||||
self.prometheus_server: typing.Optional[PrometheusServer] = None
|
||||
|
||||
self.session_mgr = LBRYSessionManager(
|
||||
env, db, bp.mempool, bp.history_cache, bp.resolve_cache, bp.resolve_outputs_cache, daemon,
|
||||
self.shutdown_event,
|
||||
on_available_callback=bp.status_server.set_available,
|
||||
on_unavailable_callback=bp.status_server.set_unavailable
|
||||
env, db, bp, daemon, self.shutdown_event
|
||||
)
|
||||
self._indexer_task = None
|
||||
|
||||
|
||||
@@ -170,16 +170,13 @@ class SessionManager:
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
def __init__(self, env: 'Env', db: LevelDB, mempool, history_cache, resolve_cache, resolve_outputs_cache,
|
||||
daemon: 'Daemon', shutdown_event: asyncio.Event,
|
||||
on_available_callback: typing.Callable[[], None], on_unavailable_callback: typing.Callable[[], None]):
|
||||
def __init__(self, env: 'Env', db: LevelDB, bp: BlockProcessor, daemon: 'Daemon', shutdown_event: asyncio.Event):
|
||||
env.max_send = max(350000, env.max_send)
|
||||
self.env = env
|
||||
self.db = db
|
||||
self.on_available_callback = on_available_callback
|
||||
self.on_unavailable_callback = on_unavailable_callback
|
||||
self.bp = bp
|
||||
self.daemon = daemon
|
||||
self.mempool = mempool
|
||||
self.mempool = bp.mempool
|
||||
self.shutdown_event = shutdown_event
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.servers: typing.Dict[str, asyncio.AbstractServer] = {}
|
||||
@@ -189,9 +186,7 @@ class SessionManager:
|
||||
self.cur_group = SessionGroup(0)
|
||||
self.txs_sent = 0
|
||||
self.start_time = time.time()
|
||||
self.history_cache = history_cache
|
||||
self.resolve_cache = resolve_cache
|
||||
self.resolve_outputs_cache = resolve_outputs_cache
|
||||
self.history_cache = self.bp.history_cache
|
||||
self.notified_height: typing.Optional[int] = None
|
||||
# Cache some idea of room to avoid recounting on each subscription
|
||||
self.subs_room = 0
|
||||
@@ -248,7 +243,7 @@ class SessionManager:
|
||||
await self.session_event.wait()
|
||||
self.session_event.clear()
|
||||
if not paused and len(self.sessions) >= max_sessions:
|
||||
self.on_unavailable_callback()
|
||||
self.bp.status_server.set_unavailable()
|
||||
self.logger.info(f'maximum sessions {max_sessions:,d} '
|
||||
f'reached, stopping new connections until '
|
||||
f'count drops to {low_watermark:,d}')
|
||||
@@ -257,7 +252,7 @@ class SessionManager:
|
||||
# Start listening for incoming connections if paused and
|
||||
# session count has fallen
|
||||
if paused and len(self.sessions) <= low_watermark:
|
||||
self.on_available_callback()
|
||||
self.bp.status_server.set_available()
|
||||
self.logger.info('resuming listening for incoming connections')
|
||||
await self._start_external_servers()
|
||||
paused = False
|
||||
@@ -538,7 +533,7 @@ class SessionManager:
|
||||
await self.start_other()
|
||||
await self._start_external_servers()
|
||||
server_listening_event.set()
|
||||
self.on_available_callback()
|
||||
self.bp.status_server.set_available()
|
||||
# Peer discovery should start after the external servers
|
||||
# because we connect to ourself
|
||||
await asyncio.wait([
|
||||
@@ -633,9 +628,8 @@ class SessionManager:
|
||||
for hashX in touched.intersection(self.mempool_statuses.keys()):
|
||||
self.mempool_statuses.pop(hashX, None)
|
||||
|
||||
# self.bp._chain_executor
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, touched.intersection_update, self.hashx_subscriptions_by_session.keys()
|
||||
self.bp._chain_executor, touched.intersection_update, self.hashx_subscriptions_by_session.keys()
|
||||
)
|
||||
|
||||
if touched or new_touched or (height_changed and self.mempool_statuses):
|
||||
@@ -872,7 +866,8 @@ class LBRYElectrumX(SessionBase):
|
||||
self.protocol_tuple = self.PROTOCOL_MIN
|
||||
self.protocol_string = None
|
||||
self.daemon = self.session_mgr.daemon
|
||||
self.db: LevelDB = self.session_mgr.db
|
||||
self.bp: BlockProcessor = self.session_mgr.bp
|
||||
self.db: LevelDB = self.bp.db
|
||||
|
||||
@classmethod
|
||||
def protocol_min_max_strings(cls):
|
||||
@@ -1013,21 +1008,21 @@ class LBRYElectrumX(SessionBase):
|
||||
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
|
||||
|
||||
async def _cached_resolve_url(self, url):
|
||||
if url not in self.session_mgr.resolve_cache:
|
||||
self.session_mgr.resolve_cache[url] = await self.loop.run_in_executor(None, self.db._resolve, url)
|
||||
return self.session_mgr.resolve_cache[url]
|
||||
if url not in self.bp.resolve_cache:
|
||||
self.bp.resolve_cache[url] = await self.loop.run_in_executor(None, self.db._resolve, url)
|
||||
return self.bp.resolve_cache[url]
|
||||
|
||||
async def claimtrie_resolve(self, *urls) -> str:
|
||||
sorted_urls = tuple(sorted(urls))
|
||||
self.session_mgr.urls_to_resolve_count_metric.inc(len(sorted_urls))
|
||||
try:
|
||||
if sorted_urls in self.session_mgr.resolve_outputs_cache:
|
||||
return self.session_mgr.resolve_outputs_cache[sorted_urls]
|
||||
if sorted_urls in self.bp.resolve_outputs_cache:
|
||||
return self.bp.resolve_outputs_cache[sorted_urls]
|
||||
rows, extra = [], []
|
||||
for url in urls:
|
||||
if url not in self.session_mgr.resolve_cache:
|
||||
self.session_mgr.resolve_cache[url] = await self._cached_resolve_url(url)
|
||||
stream, channel, repost, reposted_channel = self.session_mgr.resolve_cache[url]
|
||||
if url not in self.bp.resolve_cache:
|
||||
self.bp.resolve_cache[url] = await self._cached_resolve_url(url)
|
||||
stream, channel, repost, reposted_channel = self.bp.resolve_cache[url]
|
||||
if isinstance(channel, ResolveCensoredError):
|
||||
rows.append(channel)
|
||||
extra.append(channel.censor_row)
|
||||
@@ -1052,7 +1047,7 @@ class LBRYElectrumX(SessionBase):
|
||||
if reposted_channel:
|
||||
extra.append(reposted_channel)
|
||||
await asyncio.sleep(0)
|
||||
self.session_mgr.resolve_outputs_cache[sorted_urls] = result = await self.loop.run_in_executor(
|
||||
self.bp.resolve_outputs_cache[sorted_urls] = result = await self.loop.run_in_executor(
|
||||
None, Outputs.to_base64, rows, extra, 0, None, None
|
||||
)
|
||||
return result
|
||||
@@ -1060,7 +1055,7 @@ class LBRYElectrumX(SessionBase):
|
||||
self.session_mgr.resolved_url_count_metric.inc(len(sorted_urls))
|
||||
|
||||
async def get_server_height(self):
|
||||
return self.db.db_height
|
||||
return self.bp.height
|
||||
|
||||
async def transaction_get_height(self, tx_hash):
|
||||
self.assert_tx_hash(tx_hash)
|
||||
@@ -1472,7 +1467,7 @@ class LBRYElectrumX(SessionBase):
|
||||
if mempool_tx:
|
||||
raw_tx, block_hash = mempool_tx.raw_tx.hex(), None
|
||||
else:
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, 1)
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
|
||||
raw_tx = tx_info['hex']
|
||||
block_hash = tx_info.get('blockhash')
|
||||
if block_hash:
|
||||
@@ -1509,7 +1504,7 @@ class LBRYElectrumX(SessionBase):
|
||||
if verbose not in (True, False):
|
||||
raise RPCError(BAD_REQUEST, f'"verbose" must be a boolean')
|
||||
|
||||
return await self.daemon_request('getrawtransaction', tx_hash, int(verbose))
|
||||
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
|
||||
|
||||
def _get_merkle_branch(self, tx_hashes, tx_pos):
|
||||
"""Return a merkle branch to a transaction.
|
||||
|
||||
+40
-20
@@ -1,9 +1,19 @@
|
||||
import struct
|
||||
import hashlib
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterable, Optional, Tuple
|
||||
|
||||
import ecdsa
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives.serialization import load_der_public_key
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
|
||||
from lbry.error import InsufficientFundsError
|
||||
from lbry.crypto.hash import hash160, sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
@@ -14,11 +24,10 @@ from lbry.schema.purchase import Purchase
|
||||
from lbry.schema.support import Support
|
||||
|
||||
from .script import InputScript, OutputScript
|
||||
from .constants import COIN, NULL_HASH32
|
||||
from .constants import COIN, DUST, NULL_HASH32
|
||||
from .bcd_data_stream import BCDataStream
|
||||
from .hash import TXRef, TXRefImmutable
|
||||
from .util import ReadOnlyList
|
||||
from .bip32 import PrivateKey, PublicKey
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.wallet.account import Account
|
||||
@@ -213,8 +222,7 @@ class Output(InputOutput):
|
||||
is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None,
|
||||
sent_supports: Optional[int] = None, sent_tips: Optional[int] = None,
|
||||
received_tips: Optional[int] = None,
|
||||
channel: Optional['Output'] = None,
|
||||
private_key: Optional[PrivateKey] = None
|
||||
channel: Optional['Output'] = None, private_key: Optional[str] = None
|
||||
) -> None:
|
||||
super().__init__(tx_ref, position)
|
||||
self.amount = amount
|
||||
@@ -227,7 +235,7 @@ class Output(InputOutput):
|
||||
self.sent_tips = sent_tips
|
||||
self.received_tips = received_tips
|
||||
self.channel = channel
|
||||
self.private_key: PrivateKey = private_key
|
||||
self.private_key = private_key
|
||||
self.purchase: 'Output' = None # txo containing purchase metadata
|
||||
self.purchased_claim: 'Output' = None # resolved claim pointed to by purchase
|
||||
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
|
||||
@@ -417,15 +425,25 @@ class Output(InputOutput):
|
||||
]
|
||||
return sha256(b''.join(pieces))
|
||||
|
||||
def get_encoded_signature(self):
|
||||
signature = hexlify(self.signable.signature)
|
||||
r = int(signature[:int(len(signature)/2)], 16)
|
||||
s = int(signature[int(len(signature)/2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
|
||||
|
||||
@staticmethod
|
||||
def is_signature_valid(signature, digest, public_key_bytes):
|
||||
return PublicKey\
|
||||
.from_compressed(public_key_bytes)\
|
||||
.verify(signature, digest)
|
||||
def is_signature_valid(encoded_signature, signature_digest, public_key_bytes):
|
||||
try:
|
||||
public_key = load_der_public_key(public_key_bytes, default_backend())
|
||||
public_key.verify(encoded_signature, signature_digest, ec.ECDSA(Prehashed(hashes.SHA256())))
|
||||
return True
|
||||
except (ValueError, InvalidSignature):
|
||||
pass
|
||||
return False
|
||||
|
||||
def is_signed_by(self, channel: 'Output', ledger=None):
|
||||
return self.is_signature_valid(
|
||||
self.signable.signature,
|
||||
self.get_encoded_signature(),
|
||||
self.get_signature_digest(ledger),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
@@ -438,27 +456,29 @@ class Output(InputOutput):
|
||||
self.signable.signing_channel_hash,
|
||||
self.signable.to_message_bytes()
|
||||
]))
|
||||
self.signable.signature = channel.private_key.sign_compact(digest)
|
||||
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.script.generate()
|
||||
|
||||
def sign_data(self, data: bytes, timestamp: str) -> str:
|
||||
def sign_data(self, data:bytes, timestamp:str) -> str:
|
||||
pieces = [timestamp.encode(), self.claim_hash, data]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = self.private_key.sign_compact(digest)
|
||||
signature = self.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
return hexlify(signature).decode()
|
||||
|
||||
def clear_signature(self):
|
||||
self.channel = None
|
||||
self.signable.clear_signature()
|
||||
|
||||
def set_channel_private_key(self, private_key: PrivateKey):
|
||||
self.private_key = private_key
|
||||
self.claim.channel.public_key_bytes = private_key.public_key.pubkey_bytes
|
||||
async def generate_channel_private_key(self):
|
||||
self.private_key = await asyncio.get_event_loop().run_in_executor(
|
||||
None, ecdsa.SigningKey.generate, ecdsa.SECP256k1, None, hashlib.sha256
|
||||
)
|
||||
self.claim.channel.public_key_bytes = self.private_key.get_verifying_key().to_der()
|
||||
self.script.generate()
|
||||
return self.private_key
|
||||
|
||||
def is_channel_private_key(self, private_key: PrivateKey):
|
||||
return self.claim.channel.public_key_bytes == private_key.public_key.pubkey_bytes
|
||||
def is_channel_private_key(self, private_key):
|
||||
return self.claim.channel.public_key_bytes == private_key.get_verifying_key().to_der()
|
||||
|
||||
@classmethod
|
||||
def pay_claim_name_pubkey_hash(
|
||||
@@ -818,10 +838,10 @@ class Transaction:
|
||||
)
|
||||
if payment > cost:
|
||||
change = payment - cost
|
||||
if change > cost_of_change:
|
||||
change_amount = change - cost_of_change
|
||||
if change_amount > DUST:
|
||||
change_address = await change_account.change.get_or_create_usable_address()
|
||||
change_hash160 = change_account.ledger.address_to_hash160(change_address)
|
||||
change_amount = change - cost_of_change
|
||||
change_output = Output.pay_pubkey_hash(change_amount, change_hash160)
|
||||
change_output.is_internal_transfer = True
|
||||
tx.add_outputs([Output.pay_pubkey_hash(change_amount, change_hash160)])
|
||||
|
||||
@@ -40,17 +40,22 @@ def checkrecord(record, expected_winner, expected_claim):
|
||||
|
||||
|
||||
async def checkcontrolling(daemon: Daemon, db: SQLDB):
|
||||
records, names, futs = [], [], []
|
||||
records, claim_ids, names, futs = [], [], [], []
|
||||
for record in db.get_claims('claimtrie.claim_hash as is_controlling, claim.*', is_controlling=True):
|
||||
records.append(record)
|
||||
claim_id = hex_reverted(record['claim_hash'])
|
||||
names.append((record['normalized'], (claim_id,), "", True)) # last parameter is IncludeValues
|
||||
claim_ids.append((claim_id,))
|
||||
names.append((record['normalized'],))
|
||||
if len(names) > 50000:
|
||||
futs.append(daemon._send_vector('getclaimsfornamebyid', names))
|
||||
futs.append(daemon._send_vector('getvalueforname', names[:]))
|
||||
futs.append(daemon._send_vector('getclaimbyid', claim_ids[:]))
|
||||
names.clear()
|
||||
claim_ids.clear()
|
||||
if names:
|
||||
futs.append(daemon._send_vector('getclaimsfornamebyid', names))
|
||||
futs.append(daemon._send_vector('getvalueforname', names[:]))
|
||||
futs.append(daemon._send_vector('getclaimbyid', claim_ids[:]))
|
||||
names.clear()
|
||||
claim_ids.clear()
|
||||
|
||||
while futs:
|
||||
winners, claims = futs.pop(0), futs.pop(0)
|
||||
|
||||
@@ -18,7 +18,7 @@ ignore=words,server,rpc,schema,winpaths.py,migrator,undecorated.py
|
||||
max-parents=10
|
||||
max-args=10
|
||||
max-line-length=120
|
||||
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l,pk
|
||||
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
disable=
|
||||
c-extension-no-member,
|
||||
|
||||
@@ -7,11 +7,9 @@ BASE = os.path.dirname(__file__)
|
||||
with open(os.path.join(BASE, 'README.md'), encoding='utf-8') as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
|
||||
ROCKSDB = []
|
||||
if sys.platform.startswith('linux') or sys.platform.startswith('darwin'):
|
||||
ROCKSDB.append('lbry-rocksdb==0.8.1')
|
||||
|
||||
PLYVEL = []
|
||||
if sys.platform.startswith('linux'):
|
||||
PLYVEL.append('plyvel==1.3.0')
|
||||
|
||||
setup(
|
||||
name=__name__,
|
||||
@@ -58,8 +56,9 @@ setup(
|
||||
'attrs==18.2.0',
|
||||
'pylru==1.1.0',
|
||||
'elasticsearch==7.10.1',
|
||||
'grpcio==1.38.0'
|
||||
] + ROCKSDB,
|
||||
'grpcio==1.38.0',
|
||||
'filetype==1.0.9'
|
||||
] + PLYVEL,
|
||||
extras_require={
|
||||
'torrent': ['lbry-libtorrent'],
|
||||
'lint': ['pylint==2.10.0'],
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
#! python
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import rocksdb
|
||||
import tempfile
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
def _main(db_loc):
|
||||
opts = rocksdb.Options(create_if_missing=True)
|
||||
db = rocksdb.DB(os.path.join(db_loc, "test"), opts)
|
||||
secondary_location = os.path.join(db_loc, "secondary")
|
||||
secondary = rocksdb.DB(
|
||||
os.path.join(db_loc, "test"),
|
||||
rocksdb.Options(create_if_missing=True, max_open_files=-1),
|
||||
secondary_name=secondary_location
|
||||
)
|
||||
try:
|
||||
assert secondary.get(b"a") is None
|
||||
db.put(b"a", b"b")
|
||||
assert db.get(b"a") == b"b"
|
||||
assert secondary.get(b"a") is None
|
||||
|
||||
secondary.try_catch_up_with_primary()
|
||||
assert secondary.get(b"a") == b"b"
|
||||
finally:
|
||||
secondary.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
db_dir = tempfile.mkdtemp()
|
||||
try:
|
||||
_main(db_dir)
|
||||
log.info("rocksdb %s (%s) works!", rocksdb.__version__, rocksdb.ROCKSDB_VERSION)
|
||||
except:
|
||||
log.exception("boom")
|
||||
finally:
|
||||
shutil.rmtree(db_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+8
-18
@@ -9,7 +9,7 @@ if typing.TYPE_CHECKING:
|
||||
|
||||
|
||||
def get_time_accelerator(loop: asyncio.AbstractEventLoop,
|
||||
now: typing.Optional[float] = None) -> typing.Callable[[float], typing.Awaitable[None]]:
|
||||
instant_step: bool = False) -> typing.Callable[[float], typing.Awaitable[None]]:
|
||||
"""
|
||||
Returns an async advance() function
|
||||
|
||||
@@ -17,32 +17,22 @@ def get_time_accelerator(loop: asyncio.AbstractEventLoop,
|
||||
made by call_later, call_at, and call_soon.
|
||||
"""
|
||||
|
||||
_time = now or loop.time()
|
||||
loop.time = functools.wraps(loop.time)(lambda: _time)
|
||||
original = loop.time
|
||||
_drift = 0
|
||||
loop.time = functools.wraps(loop.time)(lambda: original() + _drift)
|
||||
|
||||
async def accelerate_time(seconds: float) -> None:
|
||||
nonlocal _time
|
||||
nonlocal _drift
|
||||
if seconds < 0:
|
||||
raise ValueError(f'Cannot go back in time ({seconds} seconds)')
|
||||
_time += seconds
|
||||
await past_events()
|
||||
_drift += seconds
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def past_events() -> None:
|
||||
while loop._scheduled:
|
||||
timer: asyncio.TimerHandle = loop._scheduled[0]
|
||||
if timer not in loop._ready and timer._when <= _time:
|
||||
loop._scheduled.remove(timer)
|
||||
loop._ready.append(timer)
|
||||
if timer._when > _time:
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def accelerator(seconds: float):
|
||||
steps = seconds * 10.0
|
||||
steps = seconds * 10.0 if not instant_step else 1
|
||||
|
||||
for _ in range(max(int(steps), 1)):
|
||||
await accelerate_time(0.1)
|
||||
await accelerate_time(seconds/steps)
|
||||
|
||||
return accelerator
|
||||
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.account import DeterministicChannelKeyManager
|
||||
|
||||
|
||||
def extract(d, keys):
|
||||
@@ -63,30 +60,15 @@ class AccountManagement(CommandTestCase):
|
||||
self.assertEqual(accounts['items'][0]['name'], 'recreated account')
|
||||
|
||||
async def test_wallet_migration(self):
|
||||
old_id, new_id, valid_key = (
|
||||
'mi9E8KqFfW5ngktU22pN2jpgsdf81ZbsGY',
|
||||
'mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8',
|
||||
'-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95'
|
||||
'466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh'
|
||||
'0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS'
|
||||
'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n'
|
||||
)
|
||||
# null certificates should get deleted
|
||||
self.account.channel_keys = {
|
||||
new_id: 'not valid key',
|
||||
'foo': 'bar',
|
||||
}
|
||||
await self.channel_create('@foo1')
|
||||
await self.channel_create('@foo2')
|
||||
await self.channel_create('@foo3')
|
||||
keys = list(self.account.channel_keys.keys())
|
||||
self.account.channel_keys[keys[0]] = None
|
||||
self.account.channel_keys[keys[1]] = "some invalid junk"
|
||||
await self.account.maybe_migrate_certificates()
|
||||
self.assertEqual(self.account.channel_keys, {})
|
||||
self.account.channel_keys = {
|
||||
new_id: 'not valid key',
|
||||
'foo': 'bar',
|
||||
'invalid address': valid_key,
|
||||
}
|
||||
await self.account.maybe_migrate_certificates()
|
||||
self.assertEqual(self.account.channel_keys, {
|
||||
new_id: valid_key
|
||||
})
|
||||
self.assertEqual(list(self.account.channel_keys.keys()), [keys[2]])
|
||||
|
||||
async def assertFindsClaims(self, claim_names, awaitable):
|
||||
self.assertEqual(claim_names, [txo.claim_name for txo in (await awaitable)['items']])
|
||||
@@ -103,7 +85,7 @@ class AccountManagement(CommandTestCase):
|
||||
second_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
|
||||
tx = await self.daemon.jsonrpc_account_send(
|
||||
'0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id), blocking=True
|
||||
'0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id)
|
||||
)
|
||||
await self.confirm_tx(tx.id)
|
||||
await self.assertOutputAmount(['0.05', '9.949876'], utxo_list())
|
||||
@@ -192,100 +174,3 @@ class AccountManagement(CommandTestCase):
|
||||
bad_address = address[0:20] + '9999999' + address[27:]
|
||||
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
|
||||
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
|
||||
|
||||
async def test_hybrid_channel_keys(self):
|
||||
# non-deterministic channel
|
||||
self.account.channel_keys = {
|
||||
'mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8':
|
||||
'-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95'
|
||||
'466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh'
|
||||
'0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS'
|
||||
'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n'
|
||||
}
|
||||
channel1 = await self.create_nondeterministic_channel('@foo1', '1.0', unhexlify(
|
||||
'3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1'
|
||||
'66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16'
|
||||
'a97a6d6a4a8effd29d748901bb9789352519cd00b13d'
|
||||
))
|
||||
await self.confirm_tx(channel1['txid'])
|
||||
|
||||
# deterministic channel
|
||||
channel2 = await self.channel_create('@foo2')
|
||||
|
||||
await self.stream_create('stream-in-channel1', '0.01', channel_id=self.get_claim_id(channel1))
|
||||
await self.stream_create('stream-in-channel2', '0.01', channel_id=self.get_claim_id(channel2))
|
||||
|
||||
resolved_stream1 = await self.resolve('@foo1/stream-in-channel1')
|
||||
self.assertEqual('stream-in-channel1', resolved_stream1['name'])
|
||||
self.assertTrue(resolved_stream1['is_channel_signature_valid'])
|
||||
|
||||
resolved_stream2 = await self.resolve('@foo2/stream-in-channel2')
|
||||
self.assertEqual('stream-in-channel2', resolved_stream2['name'])
|
||||
self.assertTrue(resolved_stream2['is_channel_signature_valid'])
|
||||
|
||||
async def test_deterministic_channel_keys(self):
|
||||
seed = self.account.seed
|
||||
keys = self.account.deterministic_channel_keys
|
||||
|
||||
# create two channels and make sure they have different keys
|
||||
channel1a = await self.channel_create('@foo1')
|
||||
channel2a = await self.channel_create('@foo2')
|
||||
self.assertNotEqual(
|
||||
channel1a['outputs'][0]['value']['public_key'],
|
||||
channel2a['outputs'][0]['value']['public_key'],
|
||||
)
|
||||
|
||||
# start another daemon from the same seed
|
||||
self.daemon2 = await self.add_daemon(seed=seed)
|
||||
channel2b, channel1b = (await self.daemon2.jsonrpc_channel_list())['items']
|
||||
|
||||
# both daemons end up with the same channel signing keys automagically
|
||||
self.assertTrue(channel1b.has_private_key)
|
||||
self.assertEqual(
|
||||
channel1a['outputs'][0]['value']['public_key_id'],
|
||||
channel1b.private_key.address
|
||||
)
|
||||
self.assertTrue(channel2b.has_private_key)
|
||||
self.assertEqual(
|
||||
channel2a['outputs'][0]['value']['public_key_id'],
|
||||
channel2b.private_key.address
|
||||
)
|
||||
|
||||
# repeatedly calling next channel key returns the same key when not used
|
||||
current_known = keys.last_known
|
||||
next_key = await keys.generate_next_key()
|
||||
self.assertEqual(current_known, keys.last_known)
|
||||
self.assertEqual(next_key.address, (await keys.generate_next_key()).address)
|
||||
# again, should be idempotent
|
||||
next_key = await keys.generate_next_key()
|
||||
self.assertEqual(current_known, keys.last_known)
|
||||
self.assertEqual(next_key.address, (await keys.generate_next_key()).address)
|
||||
|
||||
# create third channel while both daemons running, second daemon should pick it up
|
||||
channel3a = await self.channel_create('@foo3')
|
||||
self.assertEqual(current_known+1, keys.last_known)
|
||||
self.assertNotEqual(next_key.address, (await keys.generate_next_key()).address)
|
||||
channel3b, = (await self.daemon2.jsonrpc_channel_list(name='@foo3'))['items']
|
||||
self.assertTrue(channel3b.has_private_key)
|
||||
self.assertEqual(
|
||||
channel3a['outputs'][0]['value']['public_key_id'],
|
||||
channel3b.private_key.address
|
||||
)
|
||||
|
||||
# channel key cache re-populated after simulated restart
|
||||
|
||||
# reset cache
|
||||
self.account.deterministic_channel_keys = DeterministicChannelKeyManager(self.account)
|
||||
channel3c, channel2c, channel1c = (await self.daemon.jsonrpc_channel_list())['items']
|
||||
self.assertFalse(channel1c.has_private_key)
|
||||
self.assertFalse(channel2c.has_private_key)
|
||||
self.assertFalse(channel3c.has_private_key)
|
||||
|
||||
# repopulate cache
|
||||
await self.account.deterministic_channel_keys.ensure_cache_primed()
|
||||
self.assertEqual(self.account.deterministic_channel_keys.last_known, keys.last_known)
|
||||
channel3c, channel2c, channel1c = (await self.daemon.jsonrpc_channel_list())['items']
|
||||
self.assertTrue(channel1c.has_private_key)
|
||||
self.assertTrue(channel2c.has_private_key)
|
||||
self.assertTrue(channel3c.has_private_key)
|
||||
|
||||
@@ -112,7 +112,7 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
|
||||
# reorg the last block dropping our claim tx
|
||||
await self.blockchain.invalidate_block(invalidated_block_hash)
|
||||
await self.conductor.clear_mempool()
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
@@ -191,7 +191,7 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
|
||||
# reorg the last block dropping our claim tx
|
||||
await self.blockchain.invalidate_block(invalidated_block_hash)
|
||||
await self.conductor.clear_mempool()
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
|
||||
@@ -101,7 +101,12 @@ class ReconnectTests(IntegrationTestCase):
|
||||
self.ledger.network.client.transport.close()
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], 'derp')
|
||||
sendtxid = await self.send_to_address_and_wait(address1, 1.1337, 1)
|
||||
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
|
||||
# await self.ledger.resolve([], 'derp')
|
||||
# self.assertTrue(self.ledger.network.is_connected)
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 10.0) # mempool
|
||||
await self.blockchain.generate(1)
|
||||
await self.on_transaction_id(sendtxid) # confirmed
|
||||
self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine
|
||||
|
||||
await self.assertBalance(self.account, '1.1337')
|
||||
@@ -130,7 +135,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.conductor.spv_node.stop()
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await asyncio.sleep(0.2) # let it retry and fail once
|
||||
await self.conductor.spv_node.start(self.conductor.lbcwallet_node)
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
@@ -160,10 +165,8 @@ class UDPServerFailDiscoveryTest(AsyncioTestCase):
|
||||
async def test_wallet_connects_despite_lack_of_udp(self):
|
||||
conductor = Conductor()
|
||||
conductor.spv_node.udp_port = '0'
|
||||
await conductor.start_lbcd()
|
||||
self.addCleanup(conductor.stop_lbcd)
|
||||
await conductor.start_lbcwallet()
|
||||
self.addCleanup(conductor.stop_lbcwallet)
|
||||
await conductor.start_blockchain()
|
||||
self.addCleanup(conductor.stop_blockchain)
|
||||
await conductor.start_spv()
|
||||
self.addCleanup(conductor.stop_spv)
|
||||
self.assertFalse(conductor.spv_node.server.bp.status_server.is_running)
|
||||
|
||||
@@ -174,7 +174,8 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
self.merchant_address = await self.account.receiving.get_or_create_usable_address()
|
||||
daemon2 = await self.add_daemon()
|
||||
address2 = await daemon2.wallet_manager.default_account.receiving.get_or_create_usable_address()
|
||||
await self.send_to_address_and_wait(address2, 2, 1, ledger=daemon2.ledger)
|
||||
sendtxid = await self.blockchain.send_to_address(address2, 2)
|
||||
await self.confirm_tx(sendtxid, daemon2.ledger)
|
||||
|
||||
stream = await self.priced_stream('a', '1.0')
|
||||
await self.assertBalance(self.account, '9.987893')
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import json
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbry.wallet import ENCRYPT_ON_DISK
|
||||
from lbry.error import InvalidPasswordError
|
||||
@@ -23,7 +22,7 @@ class WalletCommands(CommandTestCase):
|
||||
async def test_wallet_syncing_status(self):
|
||||
address = await self.daemon.jsonrpc_address_unused()
|
||||
self.assertFalse(self.daemon.jsonrpc_wallet_status()['is_syncing'])
|
||||
await self.send_to_address_and_wait(address, 1)
|
||||
await self.blockchain.send_to_address(address, 1)
|
||||
await self.ledger._update_tasks.started.wait()
|
||||
self.assertTrue(self.daemon.jsonrpc_wallet_status()['is_syncing'])
|
||||
await self.ledger._update_tasks.done.wait()
|
||||
@@ -49,7 +48,7 @@ class WalletCommands(CommandTestCase):
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
await self.conductor.spv_node.stop(True)
|
||||
self.conductor.spv_node.port = 54320
|
||||
await self.conductor.spv_node.start(self.conductor.lbcwallet_node)
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 0)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320'])
|
||||
@@ -59,22 +58,23 @@ class WalletCommands(CommandTestCase):
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 54320)
|
||||
|
||||
async def test_sending_to_scripthash_address(self):
|
||||
bal = await self.blockchain.get_balance()
|
||||
self.assertEqual(await self.blockchain.get_balance(), '95.99973580')
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
p2sh_address1 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS)
|
||||
tx = await self.account_send('2.0', p2sh_address1)
|
||||
self.assertEqual(tx['outputs'][0]['address'], p2sh_address1)
|
||||
self.assertEqual(await self.blockchain.get_balance(), str(float(bal)+3)) # +1 lbc for confirm block
|
||||
self.assertEqual(await self.blockchain.get_balance(), '98.99973580') # +1 lbc for confirm block
|
||||
await self.assertBalance(self.account, '7.999877')
|
||||
await self.wallet_send('3.0', p2sh_address1)
|
||||
self.assertEqual(await self.blockchain.get_balance(), str(float(bal)+7)) # +1 lbc for confirm block
|
||||
self.assertEqual(await self.blockchain.get_balance(), '102.99973580') # +1 lbc for confirm block
|
||||
await self.assertBalance(self.account, '4.999754')
|
||||
|
||||
async def test_balance_caching(self):
|
||||
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
|
||||
address2 = await self.daemon.jsonrpc_address_unused(account2.id)
|
||||
await self.send_to_address_and_wait(address2, 10, 2)
|
||||
await self.ledger.tasks_are_done() # don't mess with the query count while we need it
|
||||
sendtxid = await self.blockchain.send_to_address(address2, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(1)
|
||||
|
||||
wallet_balance = self.daemon.jsonrpc_wallet_balance
|
||||
ledger = self.ledger
|
||||
@@ -89,16 +89,14 @@ class WalletCommands(CommandTestCase):
|
||||
self.assertIsNone(ledger._balance_cache.get(self.account.id))
|
||||
|
||||
query_count += 2
|
||||
balance = await wallet_balance()
|
||||
self.assertEqual(await wallet_balance(), expected)
|
||||
self.assertEqual(self.ledger.db.db.query_count, query_count)
|
||||
self.assertEqual(balance, expected)
|
||||
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0')
|
||||
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0')
|
||||
|
||||
# calling again uses cache
|
||||
balance = await wallet_balance()
|
||||
self.assertEqual(await wallet_balance(), expected)
|
||||
self.assertEqual(self.ledger.db.db.query_count, query_count)
|
||||
self.assertEqual(balance, expected)
|
||||
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(self.account.id))['total'], '10.0')
|
||||
self.assertEqual(dict_values_to_lbc(ledger._balance_cache.get(account2.id))['total'], '10.0')
|
||||
|
||||
@@ -124,7 +122,8 @@ class WalletCommands(CommandTestCase):
|
||||
wallet2 = await self.daemon.jsonrpc_wallet_create('foo', create_account=True)
|
||||
account3 = wallet2.default_account
|
||||
address3 = await self.daemon.jsonrpc_address_unused(account3.id, wallet2.id)
|
||||
await self.send_to_address_and_wait(address3, 1, 1)
|
||||
await self.confirm_tx(await self.blockchain.send_to_address(address3, 1))
|
||||
await self.generate(1)
|
||||
|
||||
account_balance = self.daemon.jsonrpc_account_balance
|
||||
wallet_balance = self.daemon.jsonrpc_wallet_balance
|
||||
@@ -154,7 +153,7 @@ class WalletCommands(CommandTestCase):
|
||||
address2 = await self.daemon.jsonrpc_address_unused(account2.id)
|
||||
|
||||
# send lbc to someone else
|
||||
tx = await self.daemon.jsonrpc_account_send('1.0', address2, blocking=True)
|
||||
tx = await self.daemon.jsonrpc_account_send('1.0', address2)
|
||||
await self.confirm_tx(tx.id)
|
||||
self.assertEqual(await account_balance(), {
|
||||
'total': '8.97741',
|
||||
@@ -187,7 +186,7 @@ class WalletCommands(CommandTestCase):
|
||||
})
|
||||
|
||||
# tip claimed
|
||||
tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0, blocking=True)
|
||||
tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0)
|
||||
await self.confirm_tx(tx.id)
|
||||
self.assertEqual(await account_balance(), {
|
||||
'total': '9.277303',
|
||||
@@ -238,7 +237,8 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
"carbon smart garage balance margin twelve"
|
||||
)
|
||||
address = (await self.daemon2.wallet_manager.default_account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
await self.send_to_address_and_wait(address, 1, 1, ledger=self.daemon2.ledger)
|
||||
sendtxid = await self.blockchain.send_to_address(address, 1)
|
||||
await self.confirm_tx(sendtxid, self.daemon2.ledger)
|
||||
|
||||
def assertWalletEncrypted(self, wallet_path, encrypted):
|
||||
with open(wallet_path) as opened:
|
||||
@@ -282,19 +282,8 @@ class WalletEncryptionAndSynchronization(CommandTestCase):
|
||||
)
|
||||
|
||||
# Channel Certificate
|
||||
# non-deterministic channel
|
||||
self.daemon2.wallet_manager.default_account.channel_keys['mqs77XbdnuxWN4cXrjKbSoGLkvAHa4f4B8'] = (
|
||||
'-----BEGIN EC PRIVATE KEY-----\nMHQCAQEEIBZRTZ7tHnYCH3IE9mCo95'
|
||||
'466L/ShYFhXGrjmSMFJw8eoAcGBSuBBAAK\noUQDQgAEmucoPz9nI+ChZrfhnh'
|
||||
'0RZ/bcX0r2G0pYBmoNKovtKzXGa8y07D66MWsW\nqXptakqO/9KddIkBu5eJNS'
|
||||
'UZzQCxPQ==\n-----END EC PRIVATE KEY-----\n'
|
||||
)
|
||||
channel = await self.create_nondeterministic_channel('@foo', '0.1', unhexlify(
|
||||
'3056301006072a8648ce3d020106052b8104000a034200049ae7283f3f6723e0a1'
|
||||
'66b7e19e1d1167f6dc5f4af61b4a58066a0d2a8bed2b35c66bccb4ec3eba316b16'
|
||||
'a97a6d6a4a8effd29d748901bb9789352519cd00b13d'
|
||||
), self.daemon2, blocking=True)
|
||||
await self.confirm_tx(channel['txid'], self.daemon2.ledger)
|
||||
channel = await daemon2.jsonrpc_channel_create('@foo', '0.1')
|
||||
await self.confirm_tx(channel.id, self.daemon2.ledger)
|
||||
|
||||
# both daemons will have the channel but only one has the cert so far
|
||||
self.assertItemCount(await daemon.jsonrpc_channel_list(), 1)
|
||||
|
||||
@@ -19,6 +19,12 @@ from lbry.crypto.hash import sha256
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
def get_encoded_signature(signature):
|
||||
signature = signature.encode() if isinstance(signature, str) else signature
|
||||
r = int(signature[:int(len(signature) / 2)], 16)
|
||||
s = int(signature[int(len(signature) / 2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
||||
|
||||
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
@@ -27,7 +33,7 @@ def verify(channel, data, signature, channel_hash=None):
|
||||
data
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
unhexlify(signature['signature']),
|
||||
get_encoded_signature(signature['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
@@ -494,7 +500,8 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
tx = await Transaction.claim_create(
|
||||
'unknown', b'{"sources":{"lbry_sd_hash":""}}', 1, address, [self.account], self.account)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
@@ -1120,17 +1127,17 @@ class ChannelCommands(CommandTestCase):
|
||||
tx = await self.channel_update(claim_id, bid='4.0')
|
||||
self.assertEqual(tx['outputs'][0]['amount'], '4.0')
|
||||
|
||||
await self.assertBalance(self.account, '5.991503')
|
||||
await self.assertBalance(self.account, '5.991447')
|
||||
|
||||
# not enough funds
|
||||
with self.assertRaisesRegex(
|
||||
InsufficientFundsError, "Not enough funds to cover this transaction."):
|
||||
await self.channel_create('@foo2', '9.0')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 1)
|
||||
await self.assertBalance(self.account, '5.991503')
|
||||
await self.assertBalance(self.account, '5.991447')
|
||||
|
||||
# spend exactly amount available, no change
|
||||
tx = await self.channel_create('@foo3', '5.981322')
|
||||
tx = await self.channel_create('@foo3', '5.981266')
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
self.assertEqual(len(tx['outputs']), 1) # no change
|
||||
self.assertItemCount(await self.daemon.jsonrpc_channel_list(), 2)
|
||||
@@ -1225,7 +1232,7 @@ class ChannelCommands(CommandTestCase):
|
||||
data_to_sign = "CAFEBABE"
|
||||
# claim new name
|
||||
await self.channel_create('@someotherchan')
|
||||
channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1', blocking=True)
|
||||
channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1')
|
||||
await self.confirm_tx(channel_tx.id)
|
||||
channel = channel_tx.outputs[0]
|
||||
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
|
||||
@@ -1248,7 +1255,7 @@ class ChannelCommands(CommandTestCase):
|
||||
await daemon2.jsonrpc_channel_import(exported_data)
|
||||
channels = (await daemon2.jsonrpc_channel_list())['items']
|
||||
self.assertEqual(1, len(channels))
|
||||
self.assertEqual(channel_private_key.private_key_bytes, channels[0].private_key.private_key_bytes)
|
||||
self.assertEqual(channel_private_key.to_string(), channels[0].private_key.to_string())
|
||||
|
||||
# second wallet can't update until channel is sent to it
|
||||
with self.assertRaisesRegex(AssertionError, 'Cannot find private key for signing output.'):
|
||||
@@ -1372,7 +1379,7 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual('8.989893', (await self.daemon.jsonrpc_account_balance())['available'])
|
||||
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id), blocking=True
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
@@ -1498,6 +1505,11 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
|
||||
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
|
||||
|
||||
await self.stream_update(repost_id, bid='0.42')
|
||||
searched_repost = (await self.claim_search(claim_id=repost_id))[0]
|
||||
self.assertEqual(searched_repost['amount'], '0.42')
|
||||
self.assertEqual(searched_repost['signing_channel']['claim_id'], spam_claim_id)
|
||||
|
||||
async def test_filtering_channels_for_removing_content(self):
|
||||
await self.channel_create('@some_channel', '0.1')
|
||||
await self.stream_create('good_content', '0.1', channel_name='@some_channel', tags=['good'])
|
||||
@@ -1554,7 +1566,7 @@ class StreamCommands(ClaimTestCase):
|
||||
)
|
||||
# test setting from env vars and starting from scratch
|
||||
await self.conductor.spv_node.stop(False)
|
||||
await self.conductor.spv_node.start(self.conductor.lbcwallet_node,
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node,
|
||||
extraconf={'BLOCKING_CHANNEL_IDS': blocking_channel_id,
|
||||
'FILTERING_CHANNEL_IDS': filtering_channel_id})
|
||||
await self.daemon.wallet_manager.reset()
|
||||
@@ -2171,7 +2183,7 @@ class SupportCommands(CommandTestCase):
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2',
|
||||
funding_account_ids=[account2.id], blocking=True)
|
||||
funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(tip['txid'])
|
||||
|
||||
@@ -2203,7 +2215,7 @@ class SupportCommands(CommandTestCase):
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2',
|
||||
funding_account_ids=[account2.id], blocking=True)
|
||||
funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(support['txid'])
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import unittest
|
||||
from unittest import skipIf
|
||||
import asyncio
|
||||
import os
|
||||
@@ -37,7 +36,8 @@ class FileCommands(CommandTestCase):
|
||||
tx_to_update.outputs[0], claim, 1, address, [self.account], self.account
|
||||
)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
|
||||
self.client_session._session.add_dht_node(('localhost', 4040))
|
||||
self.client_session.wait_start = False # fixme: this is super slow on tests
|
||||
@@ -107,18 +107,12 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
|
||||
await self.stream_create('foo', '0.01')
|
||||
stream = (await self.daemon.jsonrpc_file_list())["items"][0]
|
||||
self.assertSetEqual(
|
||||
set(await self.daemon.storage.get_blobs_to_announce()),
|
||||
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
|
||||
)
|
||||
self.assertSetEqual(set(await self.daemon.storage.get_blobs_to_announce()), {stream.sd_hash})
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True))
|
||||
# announces on download
|
||||
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
|
||||
stream = await self.daemon.jsonrpc_get('foo')
|
||||
self.assertSetEqual(
|
||||
set(await self.daemon.storage.get_blobs_to_announce()),
|
||||
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
|
||||
)
|
||||
self.assertSetEqual(set(await self.daemon.storage.get_blobs_to_announce()), {stream.sd_hash})
|
||||
|
||||
async def _purge_file(self, claim_name, full_path):
|
||||
self.assertTrue(
|
||||
@@ -222,7 +216,6 @@ class FileCommands(CommandTestCase):
|
||||
await self.wait_files_to_complete()
|
||||
self.assertNotEqual(first_path, second_path)
|
||||
|
||||
@unittest.SkipTest # FIXME: claimname/updateclaim is gone. #3480 wip, unblock #3479"
|
||||
async def test_file_list_updated_metadata_on_resolve(self):
|
||||
await self.stream_create('foo', '0.01')
|
||||
txo = (await self.daemon.resolve(self.wallet.accounts, ['lbry://foo']))['lbry://foo']
|
||||
@@ -511,7 +504,8 @@ class FileCommands(CommandTestCase):
|
||||
tx.outputs[0].claim.stream.fee.address_bytes = b''
|
||||
tx.outputs[0].script.generate()
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
async def __raw_value_update_no_fee_amount(self, tx, claim_address):
|
||||
tx = await self.daemon.jsonrpc_stream_update(
|
||||
@@ -521,7 +515,8 @@ class FileCommands(CommandTestCase):
|
||||
tx.outputs[0].claim.stream.fee.message.ClearField('amount')
|
||||
tx.outputs[0].script.generate()
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
|
||||
class DiskSpaceManagement(CommandTestCase):
|
||||
@@ -578,6 +573,11 @@ class DiskSpaceManagement(CommandTestCase):
|
||||
self.assertTrue(blobs2.issubset(blobs))
|
||||
self.assertFalse(blobs3.issubset(blobs))
|
||||
self.assertTrue(blobs4.issubset(blobs))
|
||||
# check that added_on gets set on downloads (was a bug)
|
||||
self.assertLess(0, await self.daemon.storage.run_and_return_one_or_none("select min(added_on) from blob"))
|
||||
await self.daemon.jsonrpc_file_delete(delete_all=True)
|
||||
await self.daemon.jsonrpc_get("foo4", save_file=False)
|
||||
self.assertLess(0, await self.daemon.storage.run_and_return_one_or_none("select min(added_on) from blob"))
|
||||
|
||||
|
||||
class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
@@ -616,6 +616,7 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
|
||||
|
||||
background_downloader = BackgroundDownloader(self.daemon.conf, self.daemon.storage, self.daemon.blob_manager)
|
||||
self.daemon.conf.network_storage_limit = 32
|
||||
await self.clear()
|
||||
await self.blob_clean()
|
||||
self.assertEqual(0, (await self.status())['disk_space']['total_used_mb'])
|
||||
@@ -624,6 +625,7 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
await self.blob_clean()
|
||||
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
|
||||
self.assertEqual(32, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
|
||||
self.daemon.conf.network_storage_limit = 48
|
||||
await background_downloader.download_blobs(content2)
|
||||
await self.assertBlobs(content1, content2)
|
||||
await self.blob_clean()
|
||||
@@ -653,3 +655,7 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
|
||||
self.daemon.conf.blob_storage_limit = 1
|
||||
await self.blob_clean()
|
||||
await self.assertBlobs(content1, no_files=False)
|
||||
|
||||
self.daemon.conf.network_storage_limit = 0
|
||||
await self.blob_clean()
|
||||
self.assertEqual(0, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
|
||||
@@ -80,7 +80,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
# After some soul searching Chris decides that his story needs more
|
||||
# heart and a better ending. He takes down the story and begins the rewrite.
|
||||
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=True))
|
||||
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=False))
|
||||
self.assertEqual(abandon['inputs'][0]['claim_id'], claim_id)
|
||||
await self.confirm_tx(abandon['txid'])
|
||||
|
||||
@@ -103,7 +103,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# 1 LBC to which Chris readily obliges
|
||||
ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id']
|
||||
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
|
||||
result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address, blocking=True))
|
||||
result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address))
|
||||
self.assertIn("txid", result)
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
@@ -133,7 +133,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
|
||||
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
|
||||
tx = await self.out(self.daemon.jsonrpc_support_create(
|
||||
claim_id2, '0.2', account_id=ramsey_account_id, blocking=True
|
||||
claim_id2, '0.2', account_id=ramsey_account_id
|
||||
))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
@@ -147,7 +147,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# Now he also wanted to support the original creator of the Award Winning Novel
|
||||
# So he quickly decides to send a tip to him
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id, blocking=True)
|
||||
self.daemon.jsonrpc_support_create(claim_id2, '0.3', tip=True, account_id=ramsey_account_id)
|
||||
)
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
@@ -158,7 +158,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
await self.generate(5)
|
||||
|
||||
# Seeing the ravishing success of his novel Chris adds support to his claim too
|
||||
tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4', blocking=True))
|
||||
tx = await self.out(self.daemon.jsonrpc_support_create(claim_id2, '0.4'))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
@@ -183,7 +183,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
|
||||
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
|
||||
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=True))
|
||||
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['inputs'][0]['txid'], tx['txid'])
|
||||
await self.confirm_tx(abandon['txid'])
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import hashlib
|
||||
import sys
|
||||
from bisect import bisect_right
|
||||
from binascii import hexlify, unhexlify
|
||||
from collections import defaultdict
|
||||
@@ -32,13 +31,13 @@ class BaseResolveTestCase(CommandTestCase):
|
||||
self.assertEqual(claim_from_es['effective_amount'], claim_from_db.effective_amount)
|
||||
|
||||
def assertMatchDBClaim(self, expected, claim):
|
||||
self.assertEqual(expected['claimid'], claim.claim_hash.hex())
|
||||
self.assertEqual(expected['validatheight'], claim.activation_height)
|
||||
self.assertEqual(expected['lasttakeoverheight'], claim.last_takeover_height)
|
||||
self.assertEqual(expected['txid'], claim.tx_hash[::-1].hex())
|
||||
self.assertEqual(expected['claimId'], claim.claim_hash.hex())
|
||||
self.assertEqual(expected['validAtHeight'], claim.activation_height)
|
||||
self.assertEqual(expected['lastTakeoverHeight'], claim.last_takeover_height)
|
||||
self.assertEqual(expected['txId'], claim.tx_hash[::-1].hex())
|
||||
self.assertEqual(expected['n'], claim.position)
|
||||
self.assertEqual(expected['amount'], claim.amount)
|
||||
self.assertEqual(expected['effectiveamount'], claim.effective_amount)
|
||||
self.assertEqual(expected['effectiveAmount'], claim.effective_amount)
|
||||
|
||||
async def assertResolvesToClaimId(self, name, claim_id):
|
||||
other = await self.resolve(name)
|
||||
@@ -54,10 +53,9 @@ class BaseResolveTestCase(CommandTestCase):
|
||||
self.assertEqual(claim_id, claim_from_es[0][0]['claim_hash'][::-1].hex())
|
||||
|
||||
async def assertNoClaimForName(self, name: str):
|
||||
lbrycrd_winning = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name))
|
||||
lbrycrd_winning = json.loads(await self.blockchain._cli_cmnd('getvalueforname', name))
|
||||
stream, channel, _, _ = await self.conductor.spv_node.server.bp.db.resolve(name)
|
||||
if 'claims' in lbrycrd_winning and lbrycrd_winning['claims'] is not None:
|
||||
self.assertEqual(len(lbrycrd_winning['claims']), 0)
|
||||
self.assertNotIn('claimId', lbrycrd_winning)
|
||||
if stream is not None:
|
||||
self.assertIsInstance(stream, LookupError)
|
||||
else:
|
||||
@@ -65,23 +63,20 @@ class BaseResolveTestCase(CommandTestCase):
|
||||
claim_from_es = await self.conductor.spv_node.server.bp.db.search_index.search(name=name)
|
||||
self.assertListEqual([], claim_from_es[0])
|
||||
|
||||
async def assertNoClaim(self, name: str, claim_id: str):
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getclaimsfornamebyid', name, '["' + claim_id + '"]'))
|
||||
if 'claims' in expected and expected['claims'] is not None:
|
||||
# ensure that if we do have the matching claim that it is not active
|
||||
self.assertEqual(expected['claims'][0]['effectiveamount'], 0)
|
||||
|
||||
async def assertNoClaim(self, claim_id: str):
|
||||
self.assertDictEqual(
|
||||
{}, json.loads(await self.blockchain._cli_cmnd('getclaimbyid', claim_id))
|
||||
)
|
||||
claim_from_es = await self.conductor.spv_node.server.bp.db.search_index.search(claim_id=claim_id)
|
||||
self.assertListEqual([], claim_from_es[0])
|
||||
claim = await self.conductor.spv_node.server.bp.db.fs_getclaimbyid(claim_id)
|
||||
self.assertIsNone(claim)
|
||||
|
||||
async def assertMatchWinningClaim(self, name):
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getclaimsfornamebybid', name, "[0]"))
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getvalueforname', name))
|
||||
stream, channel, _, _ = await self.conductor.spv_node.server.bp.db.resolve(name)
|
||||
claim = stream if stream else channel
|
||||
expected['claims'][0]['lasttakeoverheight'] = expected['lasttakeoverheight']
|
||||
await self._assertMatchClaim(expected['claims'][0], claim)
|
||||
await self._assertMatchClaim(expected, claim)
|
||||
return claim
|
||||
|
||||
async def _assertMatchClaim(self, expected, claim):
|
||||
@@ -91,81 +86,70 @@ class BaseResolveTestCase(CommandTestCase):
|
||||
)
|
||||
self.assertEqual(len(claim_from_es[0]), 1)
|
||||
self.assertMatchESClaim(claim_from_es[0][0], claim)
|
||||
self._check_supports(claim.claim_hash.hex(), expected.get('supports', []),
|
||||
claim_from_es[0][0]['support_amount'])
|
||||
self._check_supports(claim.claim_hash.hex(), expected['supports'], claim_from_es[0][0]['support_amount'])
|
||||
|
||||
async def assertMatchClaim(self, name, claim_id, is_active_in_lbrycrd=True):
|
||||
async def assertMatchClaim(self, claim_id, is_active_in_lbrycrd=True):
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getclaimbyid', claim_id))
|
||||
claim = await self.conductor.spv_node.server.bp.db.fs_getclaimbyid(claim_id)
|
||||
if is_active_in_lbrycrd:
|
||||
if not expected:
|
||||
self.assertIsNone(claim)
|
||||
return
|
||||
self.assertMatchDBClaim(expected, claim)
|
||||
else:
|
||||
self.assertDictEqual({}, expected)
|
||||
claim_from_es = await self.conductor.spv_node.server.bp.db.search_index.search(
|
||||
claim_id=claim.claim_hash.hex()
|
||||
)
|
||||
self.assertEqual(len(claim_from_es[0]), 1)
|
||||
self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), claim.claim_hash.hex())
|
||||
self.assertMatchESClaim(claim_from_es[0][0], claim)
|
||||
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getclaimsfornamebyid', name, '["' + claim_id + '"]'))
|
||||
if is_active_in_lbrycrd:
|
||||
if not expected:
|
||||
self.assertIsNone(claim)
|
||||
return
|
||||
expected['claims'][0]['lasttakeoverheight'] = expected['lasttakeoverheight']
|
||||
self.assertMatchDBClaim(expected['claims'][0], claim)
|
||||
self._check_supports(claim.claim_hash.hex(), expected['claims'][0].get('supports', []),
|
||||
claim_from_es[0][0]['support_amount'])
|
||||
else:
|
||||
if 'claims' in expected and expected['claims'] is not None:
|
||||
# ensure that if we do have the matching claim that it is not active
|
||||
self.assertEqual(expected['claims'][0]['effectiveamount'], 0)
|
||||
self._check_supports(
|
||||
claim.claim_hash.hex(), expected.get('supports', []), claim_from_es[0][0]['support_amount'],
|
||||
is_active_in_lbrycrd
|
||||
)
|
||||
return claim
|
||||
|
||||
async def assertMatchClaimIsWinning(self, name, claim_id):
|
||||
self.assertEqual(claim_id, (await self.assertMatchWinningClaim(name)).claim_hash.hex())
|
||||
await self.assertMatchClaimsForName(name)
|
||||
|
||||
def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount):
|
||||
total_lbrycrd_amount = 0.0
|
||||
total_es_amount = 0.0
|
||||
active_es_amount = 0.0
|
||||
def _check_supports(self, claim_id, lbrycrd_supports, es_support_amount, is_active_in_lbrycrd=True):
|
||||
total_amount = 0
|
||||
db = self.conductor.spv_node.server.bp.db
|
||||
es_supports = db.get_supports(bytes.fromhex(claim_id))
|
||||
|
||||
# we're only concerned about active supports here, and they should match
|
||||
self.assertTrue(len(es_supports) >= len(lbrycrd_supports))
|
||||
|
||||
for i, (tx_num, position, amount) in enumerate(es_supports):
|
||||
total_es_amount += amount
|
||||
valid_height = db.get_activation(tx_num, position, is_support=True)
|
||||
if valid_height > db.db_height:
|
||||
continue
|
||||
active_es_amount += amount
|
||||
txid = db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex()
|
||||
support = next(filter(lambda s: s['txid'] == txid and s['n'] == position, lbrycrd_supports))
|
||||
total_lbrycrd_amount += support['amount']
|
||||
self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num))
|
||||
self.assertEqual(support['validatheight'], valid_height)
|
||||
|
||||
self.assertEqual(total_es_amount, es_support_amount)
|
||||
self.assertEqual(active_es_amount, total_lbrycrd_amount)
|
||||
for i, (tx_num, position, amount) in enumerate(db.get_supports(bytes.fromhex(claim_id))):
|
||||
total_amount += amount
|
||||
if is_active_in_lbrycrd:
|
||||
support = lbrycrd_supports[i]
|
||||
self.assertEqual(support['txId'], db.prefix_db.tx_hash.get(tx_num, deserialize_value=False)[::-1].hex())
|
||||
self.assertEqual(support['n'], position)
|
||||
self.assertEqual(support['height'], bisect_right(db.tx_counts, tx_num))
|
||||
self.assertEqual(support['validAtHeight'], db.get_activation(tx_num, position, is_support=True))
|
||||
self.assertEqual(total_amount, es_support_amount, f"lbrycrd support amount: {total_amount} vs es: {es_support_amount}")
|
||||
|
||||
async def assertMatchClaimsForName(self, name):
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name, "", "true"))
|
||||
expected = json.loads(await self.blockchain._cli_cmnd('getclaimsforname', name))
|
||||
|
||||
db = self.conductor.spv_node.server.bp.db
|
||||
# self.assertEqual(len(expected['claims']), len(db_claims.claims))
|
||||
# self.assertEqual(expected['lastTakeoverHeight'], db_claims.lastTakeoverHeight)
|
||||
last_takeover = json.loads(await self.blockchain._cli_cmnd('getvalueforname', name))['lastTakeoverHeight']
|
||||
|
||||
for c in expected['claims']:
|
||||
c['lasttakeoverheight'] = expected['lasttakeoverheight']
|
||||
claim_id = c['claimid']
|
||||
c['lastTakeoverHeight'] = last_takeover
|
||||
claim_id = c['claimId']
|
||||
claim_hash = bytes.fromhex(claim_id)
|
||||
claim = db._fs_get_claim_by_hash(claim_hash)
|
||||
self.assertMatchDBClaim(c, claim)
|
||||
|
||||
claim_from_es = await self.conductor.spv_node.server.bp.db.search_index.search(
|
||||
claim_id=claim_id
|
||||
claim_id=c['claimId']
|
||||
)
|
||||
self.assertEqual(len(claim_from_es[0]), 1)
|
||||
self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), claim_id)
|
||||
self.assertEqual(claim_from_es[0][0]['claim_hash'][::-1].hex(), c['claimId'])
|
||||
self.assertMatchESClaim(claim_from_es[0][0], claim)
|
||||
self._check_supports(claim_id, c.get('supports', []),
|
||||
claim_from_es[0][0]['support_amount'])
|
||||
self._check_supports(c['claimId'], c['supports'], claim_from_es[0][0]['support_amount'])
|
||||
|
||||
|
||||
class ResolveCommand(BaseResolveTestCase):
|
||||
@@ -278,18 +262,18 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
self.assertEqual(claim['confirmations'], json.loads(tx_details)['confirmations'])
|
||||
|
||||
# resolve handles invalid data
|
||||
# await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
|
||||
# await self.generate(1)
|
||||
# response = await self.out(self.daemon.jsonrpc_resolve("lbry://gibberish"))
|
||||
# self.assertSetEqual({'lbry://gibberish'}, set(response))
|
||||
# claim = response['lbry://gibberish']
|
||||
# self.assertEqual(claim['name'], 'gibberish')
|
||||
# self.assertNotIn('value', claim)
|
||||
await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
|
||||
await self.generate(1)
|
||||
response = await self.out(self.daemon.jsonrpc_resolve("lbry://gibberish"))
|
||||
self.assertSetEqual({'lbry://gibberish'}, set(response))
|
||||
claim = response['lbry://gibberish']
|
||||
self.assertEqual(claim['name'], 'gibberish')
|
||||
self.assertNotIn('value', claim)
|
||||
|
||||
# resolve retries
|
||||
await self.conductor.spv_node.stop()
|
||||
resolve_task = asyncio.create_task(self.resolve('foo'))
|
||||
await self.conductor.spv_node.start(self.conductor.lbcwallet_node)
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
self.assertIsNotNone((await resolve_task)['claim_id'])
|
||||
|
||||
async def test_winning_by_effective_amount(self):
|
||||
@@ -459,16 +443,16 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
self.assertEqual(one, claim6['name'])
|
||||
|
||||
async def test_resolve_old_claim(self):
|
||||
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0', blocking=True)
|
||||
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
|
||||
await self.confirm_tx(channel.id)
|
||||
address = channel.outputs[0].get_address(self.account.ledger)
|
||||
claim = generate_signed_legacy(address, channel.outputs[0])
|
||||
tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
response = await self.resolve('@olds/example')
|
||||
self.assertTrue('is_channel_signature_valid' in response, str(response))
|
||||
self.assertTrue(response['is_channel_signature_valid'])
|
||||
|
||||
claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
|
||||
@@ -476,7 +460,8 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
'bad_example', claim.SerializeToString(), 1, address, [self.account], self.account
|
||||
)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast_and_confirm(tx)
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
response = await self.resolve('bad_example')
|
||||
self.assertFalse(response['is_channel_signature_valid'])
|
||||
@@ -661,7 +646,7 @@ class ResolveClaimTakeovers(BaseResolveTestCase):
|
||||
self.assertEqual(height, self.conductor.spv_node.server.bp.db.db_height)
|
||||
await self.assertMatchClaimIsWinning(name, winning_claim_id)
|
||||
for non_winning in non_winning_claims:
|
||||
claim = await self.assertMatchClaim(name,
|
||||
claim = await self.assertMatchClaim(
|
||||
non_winning.claim_id, is_active_in_lbrycrd=non_winning.active_in_lbrycrd
|
||||
)
|
||||
self.assertEqual(non_winning.activation_height, claim.activation_height)
|
||||
@@ -1007,7 +992,7 @@ class ResolveClaimTakeovers(BaseResolveTestCase):
|
||||
name = 'test'
|
||||
await self.generate(494)
|
||||
address = (await self.account.receiving.get_addresses(True))[0]
|
||||
await self.send_to_address_and_wait(address, 400.0)
|
||||
await self.blockchain.send_to_address(address, 400.0)
|
||||
await self.account.ledger.on_address.first
|
||||
await self.generate(100)
|
||||
self.assertEqual(800, self.conductor.spv_node.server.bp.db.db_height)
|
||||
@@ -1349,7 +1334,7 @@ class ResolveClaimTakeovers(BaseResolveTestCase):
|
||||
|
||||
await self.generate(1)
|
||||
|
||||
await self.assertMatchClaim(name, first_claim_id)
|
||||
await self.assertMatchClaim(first_claim_id)
|
||||
await self.assertMatchClaimIsWinning(name, second_claim_id)
|
||||
|
||||
async def test_remove_controlling_support(self):
|
||||
@@ -1420,12 +1405,12 @@ class ResolveClaimTakeovers(BaseResolveTestCase):
|
||||
await self.generate(32)
|
||||
|
||||
second_claim_id = (await self.stream_create(name, '0.01', allow_duplicate_name=True))['outputs'][0]['claim_id']
|
||||
await self.assertNoClaim(name, second_claim_id)
|
||||
await self.assertNoClaim(second_claim_id)
|
||||
self.assertEqual(
|
||||
len((await self.conductor.spv_node.server.bp.db.search_index.search(claim_name=name))[0]), 1
|
||||
)
|
||||
await self.generate(1)
|
||||
await self.assertMatchClaim(name, second_claim_id)
|
||||
await self.assertMatchClaim(second_claim_id)
|
||||
self.assertEqual(
|
||||
len((await self.conductor.spv_node.server.bp.db.search_index.search(claim_name=name))[0]), 2
|
||||
)
|
||||
@@ -1585,7 +1570,7 @@ class ResolveAfterReorg(BaseResolveTestCase):
|
||||
|
||||
# reorg the last block dropping our claim tx
|
||||
await self.blockchain.invalidate_block(invalidated_block_hash)
|
||||
await self.conductor.clear_mempool()
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
@@ -1664,7 +1649,7 @@ class ResolveAfterReorg(BaseResolveTestCase):
|
||||
|
||||
# reorg the last block dropping our claim tx
|
||||
await self.blockchain.invalidate_block(invalidated_block_hash)
|
||||
await self.conductor.clear_mempool()
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
@@ -1736,7 +1721,7 @@ def generate_signed_legacy(address: bytes, output: Output):
|
||||
claim.SerializeToString(),
|
||||
output.claim_hash[::-1]
|
||||
]))
|
||||
signature = output.private_key.sign_compact(digest)
|
||||
signature = output.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
claim.publisherSignature.version = 1
|
||||
claim.publisherSignature.signatureType = 1
|
||||
claim.publisherSignature.signature = signature
|
||||
|
||||
@@ -21,8 +21,9 @@ class BasicTransactionTest(IntegrationTestCase):
|
||||
[asyncio.ensure_future(self.on_address_update(address1)),
|
||||
asyncio.ensure_future(self.on_address_update(address2))]
|
||||
))
|
||||
await self.send_to_address_and_wait(address1, 5)
|
||||
await self.send_to_address_and_wait(address2, 5, 1)
|
||||
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
|
||||
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
|
||||
await self.blockchain.generate(1)
|
||||
await notifications
|
||||
|
||||
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
|
||||
@@ -31,9 +32,7 @@ class BasicTransactionTest(IntegrationTestCase):
|
||||
channel_txo = Output.pay_claim_name_pubkey_hash(
|
||||
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
|
||||
)
|
||||
channel_txo.set_channel_private_key(
|
||||
await self.account.generate_channel_private_key()
|
||||
)
|
||||
await channel_txo.generate_channel_private_key()
|
||||
channel_txo.script.generate()
|
||||
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
|
||||
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
import unittest
|
||||
|
||||
from lbry.testcase import CommandTestCase
|
||||
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_txo_dust_prevention(self):
|
||||
address = await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
tx = await self.account_send('9.9997758', address)
|
||||
# dust prevention threshold not reached, small txo created
|
||||
self.assertEqual(2, len(tx['outputs']))
|
||||
self.assertEqual(tx['outputs'][1]['amount'], '0.0001002')
|
||||
tx = await self.account_send('9.999706', address)
|
||||
# dust prevention prevented dust
|
||||
self.assertEqual(1, len(tx['outputs']))
|
||||
self.assertEqual(tx['outputs'][0]['amount'], '9.999706')
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_account_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id), blocking=True
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
|
||||
@@ -29,9 +38,10 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
self.assertFalse(result['success'])
|
||||
|
||||
async def test_utxo_release(self):
|
||||
await self.send_to_address_and_wait(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1, 1
|
||||
sendtxid = await self.blockchain.send_to_address(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1
|
||||
)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
await self.ledger.reserve_outputs(await self.account.get_utxos())
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
@@ -41,7 +51,6 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
class TestSegwit(CommandTestCase):
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_segwit(self):
|
||||
p2sh_address1 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS)
|
||||
p2sh_address2 = await self.blockchain.get_new_address(self.blockchain.P2SH_SEGWIT_ADDRESS)
|
||||
@@ -55,13 +64,14 @@ class TestSegwit(CommandTestCase):
|
||||
p2sh_txid2 = await self.blockchain.send_to_address(p2sh_address2, '1.0')
|
||||
bech32_txid1 = await self.blockchain.send_to_address(bech32_address1, '1.0')
|
||||
bech32_txid2 = await self.blockchain.send_to_address(bech32_address2, '1.0')
|
||||
|
||||
await self.generate(1)
|
||||
|
||||
# P2SH & BECH32 can pay to P2SH address
|
||||
tx = await self.blockchain.create_raw_transaction([
|
||||
{"txid": p2sh_txid1, "vout": 0},
|
||||
{"txid": bech32_txid1, "vout": 0},
|
||||
], {p2sh_address3: 1.9}
|
||||
], [{p2sh_address3: '1.9'}]
|
||||
)
|
||||
tx = await self.blockchain.sign_raw_transaction_with_wallet(tx)
|
||||
p2sh_txid3 = await self.blockchain.send_raw_transaction(tx)
|
||||
@@ -72,7 +82,7 @@ class TestSegwit(CommandTestCase):
|
||||
tx = await self.blockchain.create_raw_transaction([
|
||||
{"txid": p2sh_txid2, "vout": 0},
|
||||
{"txid": bech32_txid2, "vout": 0},
|
||||
], {bech32_address3: 1.9}
|
||||
], [{bech32_address3: '1.9'}]
|
||||
)
|
||||
tx = await self.blockchain.sign_raw_transaction_with_wallet(tx)
|
||||
bech32_txid3 = await self.blockchain.send_raw_transaction(tx)
|
||||
@@ -84,9 +94,12 @@ class TestSegwit(CommandTestCase):
|
||||
tx = await self.blockchain.create_raw_transaction([
|
||||
{"txid": p2sh_txid3, "vout": 0},
|
||||
{"txid": bech32_txid3, "vout": 0},
|
||||
], {address: 3.5}
|
||||
], [{address: '3.5'}]
|
||||
)
|
||||
tx = await self.blockchain.sign_raw_transaction_with_wallet(tx)
|
||||
txid = await self.blockchain.send_raw_transaction(tx)
|
||||
await self.generate_and_wait(1, [txid])
|
||||
await self.on_transaction_id(txid)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid)
|
||||
|
||||
await self.assertBalance(self.account, '13.5')
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
import random
|
||||
from itertools import chain
|
||||
|
||||
import lbry.wallet.rpc.jsonrpc
|
||||
from lbry.wallet.transaction import Transaction, Output, Input
|
||||
from lbry.testcase import IntegrationTestCase
|
||||
from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis
|
||||
@@ -19,10 +19,10 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
# to the 10th receiving address for a total of 30 UTXOs on the entire account
|
||||
for i in range(10):
|
||||
notification = asyncio.ensure_future(self.on_address_update(addresses[i]))
|
||||
_ = await self.send_to_address_and_wait(addresses[i], 10)
|
||||
txid = await self.blockchain.send_to_address(addresses[i], 10)
|
||||
await notification
|
||||
notification = asyncio.ensure_future(self.on_address_update(addresses[9]))
|
||||
_ = await self.send_to_address_and_wait(addresses[9], 10)
|
||||
txid = await self.blockchain.send_to_address(addresses[9], 10)
|
||||
await notification
|
||||
|
||||
# use batching to reduce issues with send_to_address on cli
|
||||
@@ -88,10 +88,12 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
await self.assertBalance(account2, '0.0')
|
||||
|
||||
addresses = await account1.receiving.get_addresses()
|
||||
txids = []
|
||||
for address in addresses[:5]:
|
||||
txids.append(await self.send_to_address_and_wait(address, 1.1))
|
||||
await self.generate_and_wait(1, txids)
|
||||
txids = await asyncio.gather(*(
|
||||
self.blockchain.send_to_address(address, 1.1) for address in addresses[:5]
|
||||
))
|
||||
await asyncio.wait([self.on_transaction_id(txid) for txid in txids]) # mempool
|
||||
await self.blockchain.generate(1)
|
||||
await asyncio.wait([self.on_transaction_id(txid) for txid in txids]) # confirmed
|
||||
await self.assertBalance(account1, '5.5')
|
||||
await self.assertBalance(account2, '0.0')
|
||||
|
||||
@@ -146,8 +148,11 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
return summary
|
||||
self.conductor.spv_node.server.bp.mempool.transaction_summaries = random_summary
|
||||
# 10 unconfirmed txs, all from blockchain wallet
|
||||
for i in range(10):
|
||||
await self.send_to_address_and_wait(address, 10)
|
||||
sends = [self.blockchain.send_to_address(address, 10) for _ in range(10)]
|
||||
# use batching to reduce issues with send_to_address on cli
|
||||
for batch in range(0, len(sends), 10):
|
||||
txids = await asyncio.gather(*sends[batch:batch + 10])
|
||||
await asyncio.wait([self.on_transaction_id(txid) for txid in txids])
|
||||
remote_status = await self.ledger.network.subscribe_address(address)
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
# 20 unconfirmed txs, 10 from blockchain, 10 from local to local
|
||||
@@ -165,7 +170,8 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
remote_status = await self.ledger.network.subscribe_address(address)
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
# server history grows unordered
|
||||
await self.send_to_address_and_wait(address, 1)
|
||||
txid = await self.blockchain.send_to_address(address, 1)
|
||||
await self.on_transaction_id(txid)
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1]))
|
||||
self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync))
|
||||
@@ -189,7 +195,7 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
self.ledger, 2000000000000, [self.account], set_reserved=False, return_insufficient_funds=True
|
||||
)
|
||||
got_amounts = [estimator.effective_amount for estimator in spendable]
|
||||
self.assertListEqual(sorted(amounts), sorted(got_amounts))
|
||||
self.assertListEqual(amounts, got_amounts)
|
||||
|
||||
async def test_sqlite_coin_chooser(self):
|
||||
wallet_manager = WalletManager([self.wallet], {self.ledger.get_id(): self.ledger})
|
||||
@@ -200,26 +206,26 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
other_account = self.wallet.generate_account(self.ledger)
|
||||
other_address = await other_account.receiving.get_or_create_usable_address()
|
||||
self.ledger.coin_selection_strategy = 'sqlite'
|
||||
await self.ledger.subscribe_account(other_account)
|
||||
await self.ledger.subscribe_account(self.account)
|
||||
|
||||
accepted = asyncio.ensure_future(self.on_address_update(address))
|
||||
_ = await self.send_to_address_and_wait(address, 1.0)
|
||||
txid = await self.blockchain.send_to_address(address, 1.0)
|
||||
await accepted
|
||||
|
||||
accepted = asyncio.ensure_future(self.on_address_update(address))
|
||||
_ = await self.send_to_address_and_wait(address, 1.0)
|
||||
txid = await self.blockchain.send_to_address(address, 1.0)
|
||||
await accepted
|
||||
|
||||
accepted = asyncio.ensure_future(self.on_address_update(address))
|
||||
_ = await self.send_to_address_and_wait(address, 3.0)
|
||||
txid = await self.blockchain.send_to_address(address, 3.0)
|
||||
await accepted
|
||||
|
||||
accepted = asyncio.ensure_future(self.on_address_update(address))
|
||||
_ = await self.send_to_address_and_wait(address, 5.0)
|
||||
txid = await self.blockchain.send_to_address(address, 5.0)
|
||||
await accepted
|
||||
|
||||
accepted = asyncio.ensure_future(self.on_address_update(address))
|
||||
_ = await self.send_to_address_and_wait(address, 10.0)
|
||||
txid = await self.blockchain.send_to_address(address, 10.0)
|
||||
await accepted
|
||||
|
||||
await self.assertBalance(self.account, '20.0')
|
||||
@@ -260,8 +266,6 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
async def broadcast(tx):
|
||||
try:
|
||||
return await real_broadcast(tx)
|
||||
except lbry.wallet.rpc.jsonrpc.RPCError:
|
||||
pass
|
||||
finally:
|
||||
e.set()
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ class StorageTest(AsyncioTestCase):
|
||||
await self.storage.add_blobs((blob_hash, length, 0, 0), finished=True)
|
||||
|
||||
async def store_fake_stream(self, stream_hash, blobs=None, file_name="fake_file", key="DEADBEEF"):
|
||||
blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", random_lbry_hash())]
|
||||
blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", 0, random_lbry_hash())]
|
||||
descriptor = StreamDescriptor(
|
||||
asyncio.get_event_loop(), self.blob_dir, file_name, key, file_name, blobs, stream_hash
|
||||
)
|
||||
@@ -95,7 +95,7 @@ class StorageTest(AsyncioTestCase):
|
||||
async def make_and_store_fake_stream(self, blob_count=2, stream_hash=None):
|
||||
stream_hash = stream_hash or random_lbry_hash()
|
||||
blobs = [
|
||||
BlobInfo(i + 1, 100, "DEADBEEF", random_lbry_hash())
|
||||
BlobInfo(i + 1, 100, "DEADBEEF", 0, random_lbry_hash())
|
||||
for i in range(blob_count)
|
||||
]
|
||||
await self.store_fake_stream(stream_hash, blobs)
|
||||
|
||||
@@ -1,60 +1,68 @@
|
||||
import contextlib
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
import asyncio
|
||||
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.conf import Config
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from unittest import skip
|
||||
|
||||
|
||||
class TestBlobAnnouncer(AsyncioTestCase):
|
||||
async def setup_node(self, peer_addresses, address, node_id):
|
||||
self.nodes: typing.Dict[int, Node] = {}
|
||||
self.advance = dht_mocks.get_time_accelerator(self.loop, self.loop.time())
|
||||
self.advance = dht_mocks.get_time_accelerator(self.loop)
|
||||
self.instant_advance = dht_mocks.get_time_accelerator(self.loop)
|
||||
self.conf = Config()
|
||||
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
|
||||
await self.storage.open()
|
||||
self.peer_manager = PeerManager(self.loop)
|
||||
self.node = Node(self.loop, self.peer_manager, node_id, 4444, 4444, 3333, address)
|
||||
await self.node.start_listening(address)
|
||||
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
|
||||
for node_id, address in peer_addresses:
|
||||
await self.add_peer(node_id, address)
|
||||
await asyncio.gather(*[self.add_peer(node_id, address) for node_id, address in peer_addresses])
|
||||
for first_peer in self.nodes.values():
|
||||
for second_peer in self.nodes.values():
|
||||
if first_peer == second_peer:
|
||||
continue
|
||||
self.add_peer_to_routing_table(first_peer, second_peer)
|
||||
self.add_peer_to_routing_table(second_peer, first_peer)
|
||||
await self.advance(0.1) # just to make pings go through
|
||||
self.node.joined.set()
|
||||
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
|
||||
self.storage = SQLiteStorage(self.conf, ":memory:", self.loop, self.loop.time)
|
||||
await self.storage.open()
|
||||
self.blob_announcer = BlobAnnouncer(self.loop, self.node, self.storage)
|
||||
|
||||
async def add_peer(self, node_id, address, add_to_routing_table=True):
|
||||
#print('add', node_id.hex()[:8], address)
|
||||
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
|
||||
await n.start_listening(address)
|
||||
self.nodes.update({len(self.nodes): n})
|
||||
if add_to_routing_table:
|
||||
self.node.protocol.add_peer(
|
||||
make_kademlia_peer(
|
||||
n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port
|
||||
)
|
||||
self.add_peer_to_routing_table(self.node, n)
|
||||
|
||||
def add_peer_to_routing_table(self, adder, being_added):
|
||||
adder.protocol.add_peer(
|
||||
make_kademlia_peer(
|
||||
being_added.protocol.node_id, being_added.protocol.external_ip, being_added.protocol.udp_port
|
||||
)
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _test_network_context(self, peer_addresses=None):
|
||||
self.peer_addresses = peer_addresses or [
|
||||
(constants.generate_id(2), '1.2.3.2'),
|
||||
(constants.generate_id(3), '1.2.3.3'),
|
||||
(constants.generate_id(4), '1.2.3.4'),
|
||||
(constants.generate_id(5), '1.2.3.5'),
|
||||
(constants.generate_id(6), '1.2.3.6'),
|
||||
(constants.generate_id(7), '1.2.3.7'),
|
||||
(constants.generate_id(8), '1.2.3.8'),
|
||||
(constants.generate_id(9), '1.2.3.9'),
|
||||
async def _test_network_context(self, peer_count=200):
|
||||
self.peer_addresses = [
|
||||
(constants.generate_id(i), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big')))
|
||||
for i in range(1, peer_count + 1)
|
||||
]
|
||||
try:
|
||||
with dht_mocks.mock_network_loop(self.loop):
|
||||
await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1))
|
||||
await self.setup_node(self.peer_addresses, '1.2.3.1', constants.generate_id(1000))
|
||||
yield
|
||||
finally:
|
||||
self.blob_announcer.stop()
|
||||
@@ -73,43 +81,58 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
)
|
||||
)
|
||||
await peer.ping()
|
||||
return peer
|
||||
return last_node
|
||||
|
||||
@skip("Something from a previous test is leaking into this test and causing it to fail intermittently")
|
||||
async def test_announce_blobs(self):
|
||||
blob1 = binascii.hexlify(b'1' * 48).decode()
|
||||
blob2 = binascii.hexlify(b'2' * 48).decode()
|
||||
|
||||
async with self._test_network_context():
|
||||
await self.storage.add_blobs((blob1, 1024), (blob2, 1024), finished=True)
|
||||
await self.storage.db.execute(
|
||||
"update blob set next_announce_time=0, should_announce=1 where blob_hash in (?, ?)",
|
||||
(blob1, blob2)
|
||||
)
|
||||
async with self._test_network_context(peer_count=100):
|
||||
await self.storage.add_blobs((blob1, 1024, 0, True), (blob2, 1024, 0, True), finished=True)
|
||||
await self.storage.add_blobs(
|
||||
*((constants.generate_id(value).hex(), 1024, 0, True) for value in range(1000, 1090)),
|
||||
finished=True)
|
||||
await self.storage.db.execute("update blob set next_announce_time=0, should_announce=1")
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertEqual(2, len(to_announce))
|
||||
self.blob_announcer.start(batch_size=1) # so it covers batching logic
|
||||
self.assertEqual(92, len(to_announce))
|
||||
self.blob_announcer.start(batch_size=10) # so it covers batching logic
|
||||
# takes 60 seconds to start, but we advance 120 to ensure it processed all batches
|
||||
await self.advance(60.0 * 2)
|
||||
ongoing_announcements = asyncio.ensure_future(self.blob_announcer.wait())
|
||||
await self.instant_advance(60.0)
|
||||
await ongoing_announcements
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertEqual(0, len(to_announce))
|
||||
self.blob_announcer.stop()
|
||||
|
||||
# as routing table pollution will cause some peers to be hard to reach, we add a tolerance for CI
|
||||
tolerance = 0.8 # at least 80% of the announcements are within the top K
|
||||
for blob in await self.storage.get_all_blob_hashes():
|
||||
distance = Distance(bytes.fromhex(blob))
|
||||
candidates = list(self.nodes.values())
|
||||
candidates.sort(key=lambda sorting_node: distance(sorting_node.protocol.node_id))
|
||||
has_it = 0
|
||||
for index, node in enumerate(candidates[:constants.K], start=1):
|
||||
if node.protocol.data_store.get_peers_for_blob(bytes.fromhex(blob)):
|
||||
has_it += 1
|
||||
else:
|
||||
logging.warning("blob %s wasnt found between the best K (%s)", blob[:8], node.protocol.node_id.hex()[:8])
|
||||
self.assertGreaterEqual(has_it, int(tolerance * constants.K))
|
||||
|
||||
|
||||
# test that we can route from a poorly connected peer all the way to the announced blob
|
||||
|
||||
await self.chain_peer(constants.generate_id(10), '1.2.3.10')
|
||||
await self.chain_peer(constants.generate_id(11), '1.2.3.11')
|
||||
await self.chain_peer(constants.generate_id(12), '1.2.3.12')
|
||||
await self.chain_peer(constants.generate_id(13), '1.2.3.13')
|
||||
await self.chain_peer(constants.generate_id(14), '1.2.3.14')
|
||||
await self.advance(61.0)
|
||||
current = len(self.nodes)
|
||||
await self.chain_peer(constants.generate_id(current + 1), '1.2.3.10')
|
||||
await self.chain_peer(constants.generate_id(current + 2), '1.2.3.11')
|
||||
await self.chain_peer(constants.generate_id(current + 3), '1.2.3.12')
|
||||
await self.chain_peer(constants.generate_id(current + 4), '1.2.3.13')
|
||||
last = await self.chain_peer(constants.generate_id(current + 5), '1.2.3.14')
|
||||
|
||||
last = self.nodes[len(self.nodes) - 1]
|
||||
search_q, peer_q = asyncio.Queue(loop=self.loop), asyncio.Queue(loop=self.loop)
|
||||
search_q.put_nowait(blob1)
|
||||
|
||||
_, task = last.accumulate_peers(search_q, peer_q)
|
||||
found_peers = await peer_q.get()
|
||||
found_peers = await asyncio.wait_for(peer_q.get(), 1.0)
|
||||
task.cancel()
|
||||
|
||||
self.assertEqual(1, len(found_peers))
|
||||
@@ -119,21 +142,13 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
|
||||
async def test_popular_blob(self):
|
||||
peer_count = 150
|
||||
addresses = [
|
||||
(constants.generate_id(i + 1), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big')))
|
||||
for i in range(peer_count)
|
||||
]
|
||||
blob_hash = b'1' * 48
|
||||
blob_hash = constants.generate_id(99999)
|
||||
|
||||
async with self._test_network_context(peer_addresses=addresses):
|
||||
async with self._test_network_context(peer_count=peer_count):
|
||||
total_seen = set()
|
||||
announced_to = self.nodes[0]
|
||||
for i in range(1, peer_count):
|
||||
node = self.nodes[i]
|
||||
kad_peer = make_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip, node.protocol.udp_port
|
||||
)
|
||||
await announced_to.protocol._add_peer(kad_peer)
|
||||
announced_to = self.nodes.pop(0)
|
||||
for i, node in enumerate(self.nodes.values()):
|
||||
self.add_peer_to_routing_table(announced_to, node)
|
||||
peer = node.protocol.get_rpc_peer(
|
||||
make_kademlia_peer(
|
||||
announced_to.protocol.node_id,
|
||||
@@ -144,15 +159,15 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
response = await peer.store(blob_hash)
|
||||
self.assertEqual(response, b'OK')
|
||||
peers_for_blob = await peer.find_value(blob_hash, 0)
|
||||
if i == 1:
|
||||
if i == 0:
|
||||
self.assertNotIn(blob_hash, peers_for_blob)
|
||||
self.assertEqual(peers_for_blob[b'p'], 0)
|
||||
else:
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), min(i - 1, constants.K))
|
||||
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i)
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), min(i, constants.K))
|
||||
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i + 1)
|
||||
if i - 1 > constants.K:
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), constants.K)
|
||||
self.assertEqual(peers_for_blob[b'p'], ((i - 1) // (constants.K + 1)) + 1)
|
||||
self.assertEqual(peers_for_blob[b'p'], (i // (constants.K + 1)) + 1)
|
||||
seen = set(peers_for_blob[blob_hash])
|
||||
self.assertEqual(len(seen), constants.K)
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), len(seen))
|
||||
@@ -167,5 +182,5 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
seen.intersection_update(page_x_set)
|
||||
total_seen.update(page_x_set)
|
||||
else:
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), i - 1)
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), 8) # we always add 8 on first page
|
||||
self.assertEqual(len(total_seen), peer_count - 2)
|
||||
@@ -12,7 +12,6 @@ from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
|
||||
class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
TIMEOUT = None # not supported as it advances time
|
||||
async def test_ping_queue_discover(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.set_debug(False)
|
||||
@@ -29,7 +28,7 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
(constants.generate_id(9), '1.2.3.9'),
|
||||
]
|
||||
with dht_mocks.mock_network_loop(loop):
|
||||
advance = dht_mocks.get_time_accelerator(loop, loop.time())
|
||||
advance = dht_mocks.get_time_accelerator(loop)
|
||||
# start the nodes
|
||||
nodes: typing.Dict[int, Node] = {
|
||||
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address)
|
||||
@@ -93,7 +92,6 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
|
||||
|
||||
class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
TIMEOUT = None # not supported as it advances time
|
||||
@unittest.SkipTest
|
||||
async def test_losing_connection(self):
|
||||
async def wait_for(check_ok, insist, timeout=20):
|
||||
@@ -131,7 +129,7 @@ class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
await asyncio.gather(*[n.joined.wait() for n in nodes])
|
||||
|
||||
node = nodes[-1]
|
||||
advance = dht_mocks.get_time_accelerator(loop, loop.time())
|
||||
advance = dht_mocks.get_time_accelerator(loop)
|
||||
await advance(500)
|
||||
|
||||
# Join the network, assert that at least the known peers are in RT
|
||||
|
||||
@@ -138,7 +138,9 @@ class TestTypesV1Compatibility(TestCase):
|
||||
channel = cert.channel
|
||||
self.assertEqual(
|
||||
channel.public_key,
|
||||
'033878b1edd4a1373149909ef03f4339f6da9c2bd2214c040fd2e530463ffe6609'
|
||||
'3056301006072a8648ce3d020106052b8104000a034200043878b1edd4a1373149909ef03f4339f6da9c2b'
|
||||
'd2214c040fd2e530463ffe66098eca14fc70b50ff3aefd106049a815f595ed5a13eda7419ad78d9ed7ae47'
|
||||
'3f17'
|
||||
)
|
||||
|
||||
def test_unsigned_with_fee(self):
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import unittest
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
|
||||
class MediaTypeTests(unittest.TestCase):
|
||||
def test_guess_media_type_from_path_only(self):
|
||||
kind = guess_media_type('/tmp/test.mkv')
|
||||
self.assertEqual(kind, ('video/x-matroska', 'video'))
|
||||
|
||||
def test_defaults_for_no_extension(self):
|
||||
kind = guess_media_type('/tmp/test')
|
||||
self.assertEqual(kind, ('application/octet-stream', 'binary'))
|
||||
|
||||
def test_defaults_for_unknown_extension(self):
|
||||
kind = guess_media_type('/tmp/test.unk')
|
||||
self.assertEqual(kind, ('application/x-ext-unk', 'binary'))
|
||||
|
||||
def test_spoofed_unknown(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_unknown.txt')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_lz4 = bytearray([0x04,0x22,0x4d,0x18])
|
||||
fd.write(bytes_lz4)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/x-ext-lz4', 'binary'))
|
||||
|
||||
def test_spoofed_known(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_known.avi')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_zip = bytearray([0x50,0x4b,0x03,0x06])
|
||||
fd.write(bytes_zip)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/zip', 'binary'))
|
||||
|
||||
def test_spoofed_synonym(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
file = os.path.join(temp_dir, 'spoofed_known.cbz')
|
||||
with open(file, 'wb') as fd:
|
||||
bytes_zip = bytearray([0x50,0x4b,0x03,0x06])
|
||||
fd.write(bytes_zip)
|
||||
fd.close()
|
||||
|
||||
kind = guess_media_type(file)
|
||||
self.assertEqual(kind, ('application/vnd.comicbook+zip', 'document'))
|
||||
@@ -97,7 +97,7 @@ class TestReflector(AsyncioTestCase):
|
||||
async def test_announces(self):
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
|
||||
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
|
||||
self.assertNotIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob set to announce")
|
||||
|
||||
async def test_result_from_disconnect_mid_sd_transfer(self):
|
||||
stop = asyncio.Event()
|
||||
|
||||
@@ -451,16 +451,16 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.stream_manager.stop()
|
||||
self.client_blob_manager.stop()
|
||||
# partial removal, only sd blob is missing.
|
||||
# in this case, we recover the sd blob while the other blobs are kept untouched as 'finished'
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, stream.sd_hash))
|
||||
for blob in stream.descriptor.blobs[:-1]:
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, blob.blob_hash))
|
||||
await self.client_blob_manager.setup()
|
||||
await self.stream_manager.start()
|
||||
self.assertEqual(1, len(self.stream_manager.streams))
|
||||
self.assertListEqual([self.sd_hash], list(self.stream_manager.streams.keys()))
|
||||
for blob_hash in [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]:
|
||||
blob_status = await self.client_storage.get_blob_status(blob_hash)
|
||||
self.assertEqual('pending', blob_status)
|
||||
self.assertEqual('finished', blob_status)
|
||||
self.assertEqual('finished', self.stream_manager.streams[self.sd_hash].status)
|
||||
|
||||
sd_blob = self.client_blob_manager.get_blob(stream.sd_hash)
|
||||
@@ -468,5 +468,24 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.assertTrue(sd_blob.get_is_verified())
|
||||
self.assertListEqual(expected_analytics_events, received_events)
|
||||
|
||||
# full removal, check that status is preserved (except sd blob, which was written)
|
||||
self.client_blob_manager.stop()
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, stream.sd_hash))
|
||||
for blob in stream.descriptor.blobs[:-1]:
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, blob.blob_hash))
|
||||
await self.client_blob_manager.setup()
|
||||
await self.stream_manager.start()
|
||||
for blob_hash in [b.blob_hash for b in stream.descriptor.blobs[:-1]]:
|
||||
blob_status = await self.client_storage.get_blob_status(blob_hash)
|
||||
self.assertEqual('pending', blob_status)
|
||||
# sd blob was recovered
|
||||
sd_blob = self.client_blob_manager.get_blob(stream.sd_hash)
|
||||
self.assertTrue(sd_blob.file_exists)
|
||||
self.assertTrue(sd_blob.get_is_verified())
|
||||
self.assertListEqual(expected_analytics_events, received_events)
|
||||
# db reflects that too
|
||||
blob_status = await self.client_storage.get_blob_status(stream.sd_hash)
|
||||
self.assertEqual('finished', blob_status)
|
||||
|
||||
def test_download_then_recover_old_sort_stream_on_startup(self):
|
||||
return self.test_download_then_recover_stream_on_startup(old_sort=True)
|
||||
@@ -0,0 +1,19 @@
|
||||
import unittest
|
||||
from lbry import utils
|
||||
|
||||
|
||||
class UtilsTestCase(unittest.TestCase):
|
||||
|
||||
def test_get_colliding_prefix_bits(self):
|
||||
self.assertEqual(
|
||||
0, utils.get_colliding_prefix_bits(0xffffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
8, utils.get_colliding_prefix_bits(0x00ffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
8, utils.get_colliding_prefix_bits(0x00ffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
self.assertEqual(
|
||||
1, utils.get_colliding_prefix_bits(0x7fffffff.to_bytes(4, "big"), 0x0000000000.to_bytes(4, "big")))
|
||||
@@ -1,11 +1,7 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from lbry.wallet import (
|
||||
Wallet, Ledger, Database, Headers,
|
||||
Account, SingleKey, HierarchicalDeterministic,
|
||||
DeterministicChannelKeyManager
|
||||
)
|
||||
from lbry.wallet import Wallet, Ledger, Database, Headers, Account, SingleKey, HierarchicalDeterministic
|
||||
|
||||
|
||||
class TestAccount(AsyncioTestCase):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from lbry.wallet.bip32 import PublicKey, PrivateKey, from_extended_key_string
|
||||
from lbry.wallet.bip32 import PubKey, PrivateKey, from_extended_key_string
|
||||
from lbry.wallet import Ledger, Database, Headers
|
||||
|
||||
from tests.unit.wallet.key_fixtures import expected_ids, expected_privkeys, expected_hardened_privkeys
|
||||
@@ -11,24 +11,24 @@ class BIP32Tests(AsyncioTestCase):
|
||||
|
||||
def test_pubkey_validation(self):
|
||||
with self.assertRaisesRegex(TypeError, 'chain code must be raw bytes'):
|
||||
PublicKey(None, None, 1, None, None, None)
|
||||
PubKey(None, None, 1, None, None, None)
|
||||
with self.assertRaisesRegex(ValueError, 'invalid chain code'):
|
||||
PublicKey(None, None, b'abcd', None, None, None)
|
||||
PubKey(None, None, b'abcd', None, None, None)
|
||||
with self.assertRaisesRegex(ValueError, 'invalid child number'):
|
||||
PublicKey(None, None, b'abcd'*8, -1, None, None)
|
||||
PubKey(None, None, b'abcd'*8, -1, None, None)
|
||||
with self.assertRaisesRegex(ValueError, 'invalid depth'):
|
||||
PublicKey(None, None, b'abcd'*8, 0, 256, None)
|
||||
PubKey(None, None, b'abcd'*8, 0, 256, None)
|
||||
with self.assertRaisesRegex(TypeError, 'pubkey must be raw bytes'):
|
||||
PublicKey(None, None, b'abcd'*8, 0, 255, None)
|
||||
PubKey(None, None, b'abcd'*8, 0, 255, None)
|
||||
with self.assertRaisesRegex(ValueError, 'pubkey must be 33 bytes'):
|
||||
PublicKey(None, b'abcd', b'abcd'*8, 0, 255, None)
|
||||
PubKey(None, b'abcd', b'abcd'*8, 0, 255, None)
|
||||
with self.assertRaisesRegex(ValueError, 'invalid pubkey prefix byte'):
|
||||
PublicKey(
|
||||
PubKey(
|
||||
None,
|
||||
unhexlify('33d1a3dc8155673bc1e2214fa493ccc82d57961b66054af9b6b653ac28eeef3ffe'),
|
||||
b'abcd'*8, 0, 255, None
|
||||
)
|
||||
pubkey = PublicKey( # success
|
||||
pubkey = PubKey( # success
|
||||
None,
|
||||
unhexlify('03d1a3dc8155673bc1e2214fa493ccc82d57961b66054af9b6b653ac28eeef3ffe'),
|
||||
b'abcd'*8, 0, 1, None
|
||||
@@ -37,7 +37,7 @@ class BIP32Tests(AsyncioTestCase):
|
||||
pubkey.child(-1)
|
||||
for i in range(20):
|
||||
new_key = pubkey.child(i)
|
||||
self.assertIsInstance(new_key, PublicKey)
|
||||
self.assertIsInstance(new_key, PubKey)
|
||||
self.assertEqual(hexlify(new_key.identifier()), expected_ids[i])
|
||||
|
||||
async def test_private_key_validation(self):
|
||||
@@ -60,7 +60,7 @@ class BIP32Tests(AsyncioTestCase):
|
||||
self.assertEqual(
|
||||
ec_point[1], 86198965946979720220333266272536217633917099472454294641561154971209433250106
|
||||
)
|
||||
self.assertEqual('bUDcmraBp2zCV3QWmVVeQaEgepbs1b2gC9', private_key.address)
|
||||
self.assertEqual('bUDcmraBp2zCV3QWmVVeQaEgepbs1b2gC9', private_key.address())
|
||||
with self.assertRaisesRegex(ValueError, 'invalid BIP32 private key child number'):
|
||||
private_key.child(-1)
|
||||
self.assertIsInstance(private_key.child(PrivateKey.HARDENED), PrivateKey)
|
||||
@@ -100,5 +100,5 @@ class BIP32Tests(AsyncioTestCase):
|
||||
ledger,
|
||||
'xpub661MyMwAqRbcF84AR8yfHoMzf4S2ct6mPJtvBtvNeyN9hBHuZ6uGJszkTSn5fQUCdz3XU17eBzFeAUwV6f'
|
||||
'iW44g14WF52fYC5J483wqQ5ZP',
|
||||
), PublicKey
|
||||
), PubKey
|
||||
)
|
||||
@@ -2,13 +2,10 @@ from binascii import unhexlify
|
||||
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from lbry.wallet.constants import CENT, NULL_HASH32
|
||||
from lbry.wallet.bip32 import PrivateKey, KeyPath
|
||||
from lbry.wallet.mnemonic import Mnemonic
|
||||
from lbry.wallet import Ledger, Database, Headers, Transaction, Input, Output
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.crypto.hash import sha256
|
||||
|
||||
|
||||
def get_output(amount=CENT, pubkey_hash=NULL_HASH32):
|
||||
return Transaction() \
|
||||
.add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \
|
||||
@@ -24,11 +21,8 @@ def get_tx():
|
||||
|
||||
|
||||
async def get_channel(claim_name='@foo'):
|
||||
seed = Mnemonic.mnemonic_to_seed(Mnemonic().make_seed(), '')
|
||||
key = PrivateKey.from_seed(Ledger, seed)
|
||||
channel_key = key.child(KeyPath.CHANNEL).child(0)
|
||||
channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(), b'abc')
|
||||
channel_txo.set_channel_private_key(channel_key)
|
||||
await channel_txo.generate_channel_private_key()
|
||||
get_tx().add_outputs([channel_txo])
|
||||
return channel_txo
|
||||
|
||||
@@ -120,109 +114,6 @@ class TestValidatingOldSignatures(AsyncioTestCase):
|
||||
|
||||
self.assertTrue(stream.is_signed_by(channel, ledger))
|
||||
|
||||
def test_another_signed_claim_made_by_ytsync(self):
|
||||
stream_tx = Transaction(unhexlify(
|
||||
b'010000000185870fabdd6bd2d57749afebc0b239e8d0ebeb6f3647d6cfcabd5ea2200ac632010000006b4'
|
||||
b'83045022100877c86de154e39f21959bc2157865071924adb7930a7a8910714f27398cd2689022074270f'
|
||||
b'074ae260fff319d5e0c030691821bc75b82ff0179898ac3eaeda4123eb01210200328f7f001f22ea25d72'
|
||||
b'ba37379e3065020c4d8371d9199dc4e3770084e26b9ffffffff0240420f0000000000fdcc05b527746865'
|
||||
b'2d637269746963616c2d6e6565642d666f722d696e646570656e64656e742d6d656469614d85050191bba'
|
||||
b'd064bdc455b9ebddeeb559686b13f027615384ec7c9d981c3c21a6e3d723a654e86bd707d21174c4f697f'
|
||||
b'5080cf367a3b2dfc059e6cc14a962631df69b9886f4d8b97cb339b14633966fd5ac7d75edacdf30ac5010'
|
||||
b'a90010a304af34d1c1467ebfc8785e2a49c7d5bec3cc6db94db858f1dcf95e4256564fba586d6e01f496d'
|
||||
b'f2a34344e021d2725ffd12197468652d637269746963616c2d6e6565642d666f722e6d703418ee97eac10'
|
||||
b'22209766964656f2f6d70343230ba13e6b667a9acef7e1b1caa88b9eb1d4680dea84b1d3e838266595805'
|
||||
b'ab3343855c20af35012f942ce0d5111ce080331a1f436f7079726967687465642028636f6e74616374207'
|
||||
b'075626c69736865722928e2e3c98d065a0908800f10b80818f314423954686520437269746963616c204e'
|
||||
b'65656420666f7220496e646570656e64656e74204d65646961207c20476c656e6e20477265656e77616c6'
|
||||
b'44af006496e636c7564657320616e20696e74726f64756374696f6e20627920546f6d20576f6f64732e20'
|
||||
b'5265636f7264656420696e204c616b65204a61636b736f6e2c2054657861732c206f6e20446563656d626'
|
||||
b'57220342c20323032312e0a0a526f6e205061756c27732074776f2063616d706169676e7320666f722070'
|
||||
b'7265736964656e7420283230303820616e64203230313229207765726520776174657273686564206d6f6'
|
||||
b'd656e747320666f72206c6962657274792d6d696e6465642070656f706c652061726f756e642074686520'
|
||||
b'776f726c642e205468652022526f6e205061756c205265766f6c7574696f6e22e2809463656e746572656'
|
||||
b'42061726f756e642068697320756e64696c75746564206d657373616765206f662070656163652c207072'
|
||||
b'6f70657274792c20616e64206d61726b657473e280946368616e6765642074686520776179206d696c6c6'
|
||||
b'96f6e732074686f756768742061626f75742074686520416d65726963616e20656d7069726520616e6420'
|
||||
b'74686520416d65726963616e2066696e616e6369616c2073797374656d2e2044722e205061756c2773206'
|
||||
b'66f637573206f6e2063656e7472616c2062616e6b696e6720616e6420666f726569676e20706f6c696379'
|
||||
b'2063617567687420706f6c6974696369616e7320616e642070756e64697473206f66662067756172642c2'
|
||||
b'0666f7263696e67207468656d20746f20736372616d626c6520666f72206578706c616e6174696f6e7320'
|
||||
b'6f66206f7572204d6964646c65204561737420706f6c69637920616e6420536f766965742d7374796c652'
|
||||
b'063656e7472616c20706c616e6e696e6720617420746865204665642e20506f6c697469637320696e2041'
|
||||
b'6d657269636120686173206e6f74206265656e207468652073616d652073696e636520746865202247697'
|
||||
b'56c69616e69206d6f6d656e742220616e642022456e6420746865204665642e222054686520526f6e2050'
|
||||
b'61756c205265766f6c7574696f6e2077617320626f7468206120706f6c69746963616c20616e642063756'
|
||||
b'c747572616c207068656e6f6d656e6f6e2e0a0a303a303020496e74726f64756374696f6e20627920546f'
|
||||
b'6d20576f6f64730a343a323720476c656e6e20477265656e77616c640a2e2e2e0a68747470733a2f2f777'
|
||||
b'7772e796f75747562652e636f6d2f77617463683f763d4e4b70706d52467673453052292a276874747073'
|
||||
b'3a2f2f7468756d626e61696c732e6c6272792e636f6d2f4e4b70706d5246767345305a046e6577735a096'
|
||||
b'3617468656472616c5a0f636f72706f72617465206d656469615a08637269746963616c5a0f676c656e6e'
|
||||
b'20677265656e77616c645a0b696e646570656e64656e745a0a6a6f75726e616c69736d5a056d656469615'
|
||||
b'a056d697365735a08706f6c69746963735a0a70726f706167616e64615a08726f6e207061756c5a057472'
|
||||
b'757468620208016d7576a9140969964db5b5744e2d2d0de797f5904efc80d02188acc8814200000000001'
|
||||
b'976a91439086597f9cfc066f4749b8bb245bf561714fda888ac00000000'
|
||||
))
|
||||
stream = stream_tx.outputs[0]
|
||||
|
||||
channel_tx = Transaction(unhexlify(
|
||||
b'01000000011d47b91b409b317e427adb87ec4b0bfc9fad2abf6ec3296f41918e4b3cb9d4e7010000006a4'
|
||||
b'7304402205e53ef7fc643ed00f0240dd1c3302b82141f481ed071cbcdd6b6ec6166ffd4e002203eb28ce6'
|
||||
b'39f80253f66ff3bf45288a60133d7f5625217d1ecf3b57da440b559f012103b852d61074eb995b702a800'
|
||||
b'f284e937ece4fea7f023beb70e6b0d1bff36d64b9ffffffff0240420f0000000000fdde01b506406d6973'
|
||||
b'65734db801001299010a583056301006072a8648ce3d020106052b8104000a034200047ddb1d639d7bdd0'
|
||||
b'953d9ab0bf9e971a632f85f9823c1d85780aa3e0a702b503c2962d00f67360e803514bf5864710925aacb'
|
||||
b'effd9597532c7e60eb21b4e3fd03223d2a3b68747470733a2f2f7468756d626e61696c732e6c6272792e6'
|
||||
b'36f6d2f62616e6e65722d55436d54362d43684b7061694956753266684549734e7451420a6d697365736d'
|
||||
b'656469614ad401466561747572656420766964656f732066726f6d20746865204d6973657320496e73746'
|
||||
b'9747574652e20546865204d6973657320496e737469747574652070726f6d6f7465732041757374726961'
|
||||
b'6e2065636f6e6f6d6963732c2066726565646f6d2c20616e6420706561636520696e20746865206c69626'
|
||||
b'572616c20696e74656c6c65637475616c20747261646974696f6e206f66204c756477696720766f6e204d'
|
||||
b'69736573207468726f7567682072657365617263682c207075626c697368696e672c20616e64206564756'
|
||||
b'36174696f6e2e52362a3468747470733a2f2f7468756d626e61696c732e6c6272792e636f6d2f55436d54'
|
||||
b'362d43684b7061694956753266684549734e74516d7576a914cd77ded2400e6569f03a2580244bb395f95'
|
||||
b'f91fc88ac344ab701000000001976a914cabdbfce726d2fda92ffe0041a4303f6c6c34cda88ac00000000'
|
||||
))
|
||||
channel = channel_tx.outputs[0]
|
||||
|
||||
ledger = Ledger({
|
||||
'db': Database(':memory:'),
|
||||
'headers': Headers(':memory:')
|
||||
})
|
||||
|
||||
self.assertTrue(stream.is_signed_by(channel, ledger))
|
||||
|
||||
def test_claim_signed_using_ecdsa_validates_with_coincurve(self):
|
||||
channel_tx = Transaction(unhexlify(
|
||||
"0100000001b91d829283c0d80cb8113d5f36b6da3dfe9df3e783f158bfb3fd1b2b178d7fc9010000006b48"
|
||||
"3045022100f4e2b4ee38388c3d3a62f4b12fdd413f6f140168e85884bbeb33a3f2d3159ef502201721200f"
|
||||
"4a4f3b87484d4f47c9054e31cd3ba451dd3886a7f9f854893e7c8cf90121023f9e906e0c120f3bf74feb40"
|
||||
"f01ddeafbeb1856d91938c3bef25bed06767247cffffffff0200e1f5050000000081b505406368616e4c5d"
|
||||
"00125a0a583056301006072a8648ce3d020106052b8104000a03420004d7fa13fd8e57f3a0b878eaaf3d17"
|
||||
"9144d25ddbe4a3e4440a661f51b4134c6a13c9c98678ff8411932e60fd97d7baf03ea67ebcc21097230cfb"
|
||||
"2241348aadb55e6d7576a9149c6d700f89c77f0e8c650ba05656f8f2392782d388acf47c95350000000019"
|
||||
"76a914d9502233e0e1fc76e13e36c546f704c3124d5eaa88ac00000000"
|
||||
))
|
||||
channel = channel_tx.outputs[0]
|
||||
|
||||
stream_tx = Transaction(unhexlify(
|
||||
"010000000116a1d90763f2e3a2348c7fb438a23f232b15e3ffe3f058c3b2ab52c8bed8dcb5010000006b48"
|
||||
"30450221008f38561b3a16944c63b4f4f1562f1efe1b2060f31d249e234003ee5e3461756f02205773c99e"
|
||||
"83c968728e4f2433a13871c6ad23f6c10368ac52fa62a09f3f7ef5fd012102597f39845b98e2415b777aa0"
|
||||
"3849d346d287af7970deb05f11214b3418ae9d82ffffffff0200e1f50500000000fd0c01b505636c61696d"
|
||||
"4ce8012e6e40fa5fee1b915af3b55131dcbcebee34ab9148292b084ce3741f2e0db49783f3d854ac885f2b"
|
||||
"6304a76ef7048046e338dd414ba4c64e8468651768ffaaf550c8560637ac8c477ea481ac2a9264097240f4"
|
||||
"ab0a90010a8d010a3056bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454"
|
||||
"f4edd1373e2b64ee2e68350d916e120b746d706c69647879363171180322186170706c69636174696f6e2f"
|
||||
"6f637465742d73747265616d3230f293f5acf4310562d4a41f6620167fe6d83761a98d36738908ce5c8776"
|
||||
"1642710e55352a396276a42eda92ff5856f46f6d7576a91434bd3dc4c45cc0635eb2ad5da658727e5442ca"
|
||||
"0f88ace82f902f000000001976a91427b27c89eaebf68d063c107241584c07e5a6ccc688ac00000000"
|
||||
))
|
||||
stream = stream_tx.outputs[0]
|
||||
|
||||
ledger = Ledger({'db': Database(':memory:'), 'headers': Headers(':memory:')})
|
||||
self.assertTrue(stream.is_signed_by(channel, ledger))
|
||||
|
||||
|
||||
class TestValidateSignContent(AsyncioTestCase):
|
||||
|
||||
@@ -230,10 +121,13 @@ class TestValidateSignContent(AsyncioTestCase):
|
||||
some_content = "MEANINGLESS CONTENT AEE3353320".encode()
|
||||
timestamp_str = "1630564175"
|
||||
channel = await get_channel()
|
||||
stream = get_stream()
|
||||
signature = channel.sign_data(some_content, timestamp_str)
|
||||
stream.signable.signature = unhexlify(signature.encode())
|
||||
encoded_signature = stream.get_encoded_signature()
|
||||
pieces = [timestamp_str.encode(), channel.claim_hash, some_content]
|
||||
self.assertTrue(Output.is_signature_valid(
|
||||
unhexlify(signature.encode()),
|
||||
encoded_signature,
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
))
|
||||
Reference in new issue
Block a user