Compare commits

...
Author SHA1 Message Date
Victor Shyba a2102b91a0 add timestamps to checkpoints and estimate from chunk boundaries 2020-04-21 02:26:42 -03:00
Victor Shyba 37129bce58 use utc for date field 2020-04-21 02:22:45 -03:00
Lex Berezhny 51f573f1ea v0.70.0 2020-04-20 13:37:40 -04:00
Jack Robison c8b7cd8862 Merge pull request #2920 from lbryio/uploading-to-reflector-status
add `uploading_to_reflector` to `file_list` results
2020-04-20 13:35:02 -04:00
Jack Robison 21c112d059 lbrycrd url 2020-04-20 12:16:31 -04:00
Jack Robison 9432e1b5b2 add uploading_to_reflector to file_list results 2020-04-20 11:57:09 -04:00
Lex Berezhny 084f0ebdab v0.69.1 2020-04-17 12:55:07 -04:00
Lex Berezhny 58640c1521 Merge pull request #2917 from lbryio/faster-repost-search-query
improve how claim search query handles reposts
2020-04-17 12:53:50 -04:00
Jack Robison 7ffdfd12f8 faster not tags 2020-04-17 10:47:01 -04:00
Jack Robison cb9a30f285 faster query 2020-04-17 10:47:01 -04:00
Lex Berezhny e48bef809f Merge pull request #2919 from lbryio/exclude_internal_transfers_fix
fix issue with `--exclude_internal_transfers` where it was filtering out sent payments
2020-04-16 20:26:37 -04:00
Lex Berezhny f5d7570102 fix issue with --exclude_internal_transfers where it was filtering out sent payments 2020-04-16 17:55:49 -04:00
Lex Berezhny a600c60cf8 v0.69.0 2020-04-13 15:36:27 -04:00
Jack Robison 695eabd026 Merge pull request #2911 from lbryio/file_list_claim_ids
support `claim_id`, `channel_claim_id`, and `outpoint` args given as lists of values to match in `file_list`
2020-04-13 14:58:13 -04:00
Jack Robison e81b51a647 support claim_id, channel_claim_id, and outpoint args in file_list being lists 2020-04-13 13:19:25 -04:00
Jack Robison b7e95ff090 Merge pull request #2912 from lbryio/reflector-progress
add `reflector_progress` to `file_list` results
2020-04-13 13:00:59 -04:00
Jack Robison 3ca41be686 add reflector_progress to file_list results 2020-04-13 12:08:22 -04:00
Lex Berezhny 3152046173 Merge pull request #2910 from osilkin98/comment-schema-update
Comment_hide update
2020-04-13 10:34:30 -04:00
Lex Berezhny 3a98dc8a95 run apt-get update before installing 2020-04-13 10:14:52 -04:00
Lex Berezhny d737b28916 trying actions/checkout v2 2020-04-13 09:57:59 -04:00
Oleg Silkin 97c0dac876 linter 2020-04-07 19:28:26 -04:00
Oleg Silkin 006494b1fa hide_comments now returns lists for both hidden and visible comments 2020-04-07 19:17:27 -04:00
18 changed files with 265 additions and 53 deletions

No files matched your search

+7 -5
View File
@@ -6,7 +6,7 @@ jobs:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -17,7 +17,7 @@ jobs:
name: "tests / unit"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
@@ -37,12 +37,14 @@ jobs:
- blockchain
- other
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: sudo apt install -y --no-install-recommends ffmpeg
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ffmpeg
- run: pip install tox-travis
- run: tox -e ${{ matrix.test }}
@@ -57,7 +59,7 @@ jobs:
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '3.7'
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.68.0"
__version__ = "0.70.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+13 -11
View File
@@ -1917,9 +1917,8 @@ class Daemon(metaclass=JSONRPCServerType):
"""
@requires(STREAM_MANAGER_COMPONENT)
async def jsonrpc_file_list(
self, sort=None, reverse=False, comparison=None,
wallet_id=None, page=None, page_size=None, **kwargs):
async def jsonrpc_file_list(self, sort=None, reverse=False, comparison=None, wallet_id=None, page=None,
page_size=None, **kwargs):
"""
List files limited by optional filters
@@ -1940,17 +1939,17 @@ class Daemon(metaclass=JSONRPCServerType):
--stream_hash=<stream_hash> : (str) get file with matching stream hash
--rowid=<rowid> : (int) get file with matching row id
--added_on=<added_on> : (int) get file with matching time of insertion
--claim_id=<claim_id> : (str) get file with matching claim id
--outpoint=<outpoint> : (str) get file with matching claim outpoint
--claim_id=<claim_id> : (str) get file with matching claim id(s)
--outpoint=<outpoint> : (str) get file with matching claim outpoint(s)
--txid=<txid> : (str) get file with matching claim txid
--nout=<nout> : (int) get file with matching claim nout
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id(s)
--channel_name=<channel_name> : (str) get file with matching channel name
--claim_name=<claim_name> : (str) get file with matching claim name
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le)
--comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in)
--page=<page> : (int) page to return during paginating
--page_size=<page_size> : (int) number of items on page during pagination
--wallet_id=<wallet_id> : (str) add purchase receipts from this wallet
@@ -1960,6 +1959,7 @@ class Daemon(metaclass=JSONRPCServerType):
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
sort = sort or 'rowid'
comparison = comparison or 'eq'
paginated = paginate_list(
self.stream_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size
)
@@ -5154,10 +5154,11 @@ class Daemon(metaclass=JSONRPCServerType):
--comment_ids=<comment_ids> : (str, list) one or more comment_id to hide.
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
Returns:
(dict) keyed by comment_id, containing success info
'<comment_id>': {
"hidden": (bool) flag indicating if comment_id was hidden
Returns: lists containing the ids comments that are hidden and visible.
{
"hidden": (list) IDs of hidden comments.
"visible": (list) IDs of visible comments.
}
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
@@ -5168,6 +5169,7 @@ class Daemon(metaclass=JSONRPCServerType):
comments = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_comments_by_id', comment_ids=comment_ids
)
comments = comments['items']
claim_ids = {comment['claim_id'] for comment in comments}
claims = {cid: await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id=cid) for cid in claim_ids}
pieces = []
+6 -2
View File
@@ -108,7 +108,9 @@ def encode_file_doc():
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name'
'claim_name': '(str) None if claim is not found else the claim name',
'reflector_progress': '(int) reflector upload progress, 0 to 100',
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
}
@@ -307,7 +309,9 @@ class JSONResponseEncoder(JSONEncoder):
'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
'is_fully_reflected': managed_stream.is_fully_reflected
'is_fully_reflected': managed_stream.is_fully_reflected,
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
}
def encode_claim(self, claim):
+9 -2
View File
@@ -65,6 +65,7 @@ class ManagedStream:
'downloader',
'analytics_manager',
'fully_reflected',
'reflector_progress',
'file_output_task',
'delayed_stop_task',
'streaming_responses',
@@ -73,7 +74,8 @@ class ManagedStream:
'saving',
'finished_writing',
'started_writing',
'finished_write_attempt'
'finished_write_attempt',
'uploading_to_reflector'
]
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
@@ -101,6 +103,8 @@ class ManagedStream:
self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop)
self.reflector_progress = 0
self.uploading_to_reflector = False
self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
@@ -430,6 +434,7 @@ class ManagedStream:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
self.uploading_to_reflector = True
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
@@ -445,9 +450,10 @@ class ManagedStream:
]
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
self.claim_name, self.claim_id)
for blob_hash in we_have:
for i, blob_hash in enumerate(we_have):
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError):
return sent
except ConnectionRefusedError:
@@ -455,6 +461,7 @@ class ManagedStream:
finally:
if protocol.transport:
protocol.transport.close()
self.uploading_to_reflector = False
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+30 -1
View File
@@ -23,6 +23,9 @@ if typing.TYPE_CHECKING:
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.wallet.transaction import Transaction
from lbry.wallet.manager import WalletManager
from lbry.wallet.wallet import Wallet
log = logging.getLogger(__name__)
@@ -46,6 +49,12 @@ FILTER_FIELDS = [
'blobs_in_stream'
]
SET_FILTER_FIELDS = {
"claim_ids": "claim_id",
"channel_claim_ids": "channel_claim_id",
"outpoints": "outpoint"
}
COMPARISON_OPERATORS = {
'eq': lambda a, b: a == b,
'ne': lambda a, b: a != b,
@@ -53,6 +62,7 @@ COMPARISON_OPERATORS = {
'l': lambda a, b: a < b,
'ge': lambda a, b: a >= b,
'le': lambda a, b: a <= b,
'in': lambda a, b: a in b
}
@@ -276,15 +286,34 @@ class StreamManager:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in FILTER_FIELDS:
raise ValueError(f"'{search}' is not a valid search operation")
compare_sets = {}
if isinstance(search_by.get('claim_id'), list):
compare_sets['claim_ids'] = search_by.pop('claim_id')
if isinstance(search_by.get('outpoint'), list):
compare_sets['outpoints'] = search_by.pop('outpoint')
if isinstance(search_by.get('channel_claim_id'), list):
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
if search_by:
comparison = comparison or 'eq'
streams = []
for stream in self.streams.values():
matched = False
for set_search, val in compare_sets.items():
if COMPARISON_OPERATORS[comparison](getattr(stream, SET_FILTER_FIELDS[set_search]), val):
streams.append(stream)
matched = True
break
if matched:
continue
for search, val in search_by.items():
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
this_stream = getattr(stream, search)
if COMPARISON_OPERATORS[comparison](this_stream, val):
streams.append(stream)
break
else:
+1 -1
View File
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.4/lbrycrd-linux-1744.zip'
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
+96
View File
@@ -740,4 +740,100 @@ HASHES = {
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
}
CHUNK_TIMES = [
1446058291, 1466796062, 1466957160, 1467112807, 1467272478, 1467432743, 1467593826, 1467753971, 1467911127,
1468070719, 1468224507, 1468384171, 1468543503, 1468703272, 1468863520, 1469023822, 1469183471, 1469342714,
1469501778, 1469667089, 1469827045, 1469989357, 1470149385, 1470308706, 1470467480, 1470628399, 1470789513,
1470949409, 1471110369, 1471271233, 1471432048, 1471589330, 1471749739, 1471910365, 1472071525, 1472235982,
1472398890, 1472558207, 1472721074, 1472885421, 1473046344, 1473205609, 1473368205, 1473533441, 1473692268,
1473853245, 1474016588, 1474178224, 1474340711, 1474502952, 1474662243, 1474821799, 1474981569, 1475145919,
1475307424, 1475467443, 1475630023, 1475791989, 1475952599, 1476111994, 1476271960, 1476433228, 1476595751,
1476756365, 1476918643, 1477080426, 1477244266, 1477404425, 1477564644, 1477729818, 1477890614, 1478052281,
1478213943, 1478372873, 1478531715, 1478694298, 1478856902, 1479021042, 1479181793, 1479344484, 1479506486,
1479663432, 1479825937, 1479985117, 1480148599, 1480310388, 1480474658, 1480639077, 1480800300, 1480960405,
1481122737, 1481282632, 1481445451, 1481605639, 1481767227, 1481926963, 1482091620, 1482252291, 1482411821,
1482573703, 1482736912, 1482897678, 1483058633, 1483218595, 1483378674, 1483541215, 1483702499, 1483863896,
1484022728, 1484180459, 1484342882, 1484507416, 1484670823, 1484829895, 1484993458, 1485156047, 1485317583,
1485481218, 1485643245, 1485805928, 1485967452, 1486133206, 1486293958, 1486456274, 1486617735, 1486778872,
1486940299, 1487107013, 1487273582, 1487438229, 1487599911, 1487759331, 1487921967, 1488080929, 1488241128,
1488404512, 1488565344, 1488725701, 1488887005, 1489047968, 1489207675, 1489371326, 1489535725, 1489703338,
1489866064, 1490029222, 1490191595, 1490351274, 1490512577, 1490671939, 1490829835, 1490994376, 1491154639,
1491317939, 1491481515, 1491643279, 1491804437, 1491964639, 1492124263, 1492284273, 1492445688, 1492605884,
1492765009, 1492927220, 1493088404, 1493251395, 1493412278, 1493573461, 1493732746, 1493893060, 1494053987,
1494216326, 1494376404, 1494540073, 1494702026, 1494862527, 1495024065, 1495185322, 1495344568, 1495506617,
1495666173, 1495826079, 1495987151, 1496148961, 1496309855, 1496473442, 1496634935, 1496796598, 1496961204,
1497121769, 1497283066, 1497443548, 1497603288, 1497767555, 1497933761, 1498094921, 1498256760, 1498420931,
1498584545, 1498749559, 1498913792, 1499077566, 1499239970, 1499404091, 1499566804, 1499731157, 1499895613,
1500059235, 1500222621, 1500388811, 1500554678, 1500721542, 1500884471, 1501044844, 1501209905, 1501371165,
1501530890, 1501694185, 1501857382, 1502018129, 1502184292, 1502350341, 1502511801, 1502676243, 1502838811,
1503002137, 1503163638, 1503325023, 1503486684, 1503650746, 1503812386, 1503973244, 1504133666, 1504297628,
1504459991, 1504623358, 1504784832, 1504946961, 1505110359, 1505273975, 1505437560, 1505598954, 1505757098,
1505917242, 1506077971, 1506237591, 1506398935, 1506559037, 1506720796, 1506885747, 1507050316, 1507212971,
1507375409, 1507537815, 1507702332, 1507864531, 1508025612, 1508185476, 1508345744, 1508508500, 1508670141,
1508832224, 1508992717, 1509152807, 1509310842, 1509473481, 1509633634, 1509796495, 1509954849, 1510116079,
1510277697, 1510439798, 1510600524, 1510761373, 1510921787, 1511081683, 1511243190, 1511403276, 1511563575,
1511723540, 1511884418, 1512045789, 1512207376, 1512368713, 1512530546, 1512691400, 1512852021, 1513012694,
1513173368, 1513334988, 1513495509, 1513657679, 1513817134, 1513978820, 1514142957, 1514304599, 1514465234,
1514624591, 1514785874, 1514945051, 1515108188, 1515268796, 1515430685, 1515590234, 1515751217, 1515910476,
1516068662, 1516228676, 1516387357, 1516547616, 1516707229, 1516867094, 1517026140, 1517186251, 1517345155,
1517505393, 1517664993, 1517824788, 1517983415, 1518144196, 1518303260, 1518464979, 1518625788, 1518785521,
1518945093, 1519103570, 1519263344, 1519423481, 1519584426, 1519746582, 1519907472, 1520066218, 1520226493,
1520385895, 1520543541, 1520702453, 1520861359, 1521018892, 1521177734, 1521341622, 1521502375, 1521662214,
1521822716, 1521982142, 1522142729, 1522301856, 1522459931, 1522620631, 1522780689, 1522940183, 1523099791,
1523258399, 1523420479, 1523579443, 1523739555, 1523899500, 1524059916, 1524219172, 1524378005, 1524537757,
1524701031, 1524861748, 1525021162, 1525180528, 1525341200, 1525501213, 1525662981, 1525823008, 1525982484,
1526142364, 1526300429, 1526459717, 1526620687, 1526781127, 1526940119, 1527099136, 1527259200, 1527418437,
1527579937, 1527739330, 1527898324, 1528059125, 1528218736, 1528379347, 1528540251, 1528700525, 1528859406,
1529018377, 1529178147, 1529339191, 1529498669, 1529656834, 1529815415, 1529977050, 1530135713, 1530295787,
1530456752, 1530616682, 1530776198, 1530937477, 1531097196, 1531259398, 1531419197, 1531579213, 1531739816,
1531899928, 1532060554, 1532220789, 1532378546, 1532537193, 1532695877, 1532852995, 1533013229, 1533170401,
1533329105, 1533488406, 1533646944, 1533807289, 1533965434, 1534125093, 1534284340, 1534442620, 1534600547,
1534759292, 1534920427, 1535082093, 1535241742, 1535404226, 1535566494, 1535726910, 1535884899, 1536042729,
1536201423, 1536361908, 1536520607, 1536679327, 1536840381, 1536999216, 1537157660, 1537318284, 1537477213,
1537635506, 1537793639, 1537953780, 1538117759, 1538278913, 1538439538, 1538597977, 1538756382, 1538915421,
1539077166, 1539240317, 1539400708, 1539562594, 1539722205, 1539881920, 1540042540, 1540202519, 1540363464,
1540521096, 1540681043, 1540838897, 1541000630, 1541159754, 1541320837, 1541479768, 1541640101, 1541798960,
1541957879, 1542114653, 1542274838, 1542433802, 1542592336, 1542752146, 1542913026, 1543072942, 1543231105,
1543390483, 1543548911, 1543705884, 1543864964, 1544025570, 1544185854, 1544343817, 1544503407, 1544661411,
1544819366, 1544980849, 1545142380, 1545303819, 1545464301, 1545624146, 1545783061, 1545942620, 1546102174,
1546262550, 1546421421, 1546580475, 1546741261, 1546901871, 1547061606, 1547221600, 1547379723, 1547538785,
1547697419, 1547857490, 1548017070, 1548176553, 1548335542, 1548495290, 1548656463, 1548815906, 1548976481,
1549134901, 1549293055, 1549453525, 1549615159, 1549775541, 1549934636, 1550095275, 1550255002, 1550416229,
1550577697, 1550739302, 1550897867, 1551057080, 1551218892, 1551379080, 1551538880, 1551700931, 1551860435,
1552018109, 1552178077, 1552337707, 1552497354, 1552659923, 1552821230, 1552980036, 1553138910, 1553298180,
1553457714, 1553616756, 1553777086, 1553937586, 1554098177, 1554257525, 1554415929, 1554574675, 1554734734,
1554895337, 1555054185, 1555214104, 1555375518, 1555532785, 1555694268, 1555851788, 1556013303, 1556174215,
1556332761, 1556494320, 1556655155, 1556816169, 1556975478, 1557132393, 1557293583, 1557451564, 1557612602,
1557772604, 1557931689, 1558091515, 1558250159, 1558409668, 1558568765, 1558726947, 1558886991, 1559046885,
1559205383, 1559364094, 1559524528, 1559686227, 1559844138, 1560003768, 1560163656, 1560324916, 1560484561,
1560645681, 1560808735, 1560971999, 1561135851, 1561297483, 1561455206, 1561614235, 1561777714, 1561945864,
1562118106, 1562278703, 1562437641, 1562603277, 1562767483, 1562927551, 1563087487, 1563248549, 1563410262,
1563573257, 1563733848, 1563894208, 1564054018, 1564212252, 1564374334, 1564532357, 1564693415, 1564854450,
1565015965, 1565175795, 1565339141, 1565497044, 1565655699, 1565818978, 1565981118, 1566141639, 1566302976,
1566465005, 1566625379, 1566786170, 1566946313, 1567108918, 1567270890, 1567430354, 1567588684, 1567749712,
1567908836, 1568068607, 1568229349, 1568390117, 1568549698, 1568709328, 1568867513, 1569025318, 1569185425,
1569344544, 1569503324, 1569664889, 1569823995, 1569986419, 1570146118, 1570306012, 1570467311, 1570628581,
1570787594, 1570948886, 1571110097, 1571270823, 1571430280, 1571588101, 1571750508, 1571911503, 1572071265,
1572232449, 1572392837, 1572553922, 1572713730, 1572875606, 1573036488, 1573196968, 1573362152, 1573521448,
1573683075, 1573843849, 1574003422, 1574162840, 1574323813, 1574484633, 1574643754, 1574807310, 1574966691,
1575125631, 1575285152, 1575444009, 1575605511, 1575765796, 1575924265, 1576087060, 1576246324, 1576404707,
1576562942, 1576724468, 1576885925, 1577045396, 1577204522, 1577365567, 1577524358, 1577684296, 1577847223,
1578008388, 1578166426, 1578327792, 1578489088, 1578650676, 1578809573, 1578967948, 1579125782, 1579285861,
1579449027, 1579608917, 1579768221, 1579927859, 1580086975, 1580247978, 1580408360, 1580569255, 1580728734,
1580888441, 1581047850, 1581208990, 1581369823, 1581528564, 1581689593, 1581848711, 1582008320, 1582169619,
1582329254, 1582490606, 1582653889, 1582813888, 1582976432, 1583137185, 1583299285, 1583463034, 1583625464,
1583786150, 1583949616, 1584109878, 1584268907, 1584428799, 1584589342, 1584749487, 1584911825, 1585073681,
1585234424, 1585394114, 1585556112, 1585716022, 1585876422, 1586036811, 1586196631, 1586356275, 1586519756,
1586678956, 1586838190, 1587004938, 1587160029
]
+2 -1
View File
@@ -733,8 +733,9 @@ class Database(SQLiteMixin):
include_is_my_input = True
constraints['exclude_internal_payments__or'] = {
'txo.txo_type__not': TXO_TYPES['other'],
'txo.address__not_in': my_addresses,
'txi.address__is_null': True,
'txi.address__not_in': my_addresses
'txi.address__not_in': my_addresses,
}
sql = [f"SELECT {cols} FROM txo JOIN tx ON (tx.txid=txo.txid)"]
if is_spent:
+8 -2
View File
@@ -13,7 +13,7 @@ from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.wallet.util import ArithUint256, date_to_julian_day
from .checkpoints import HASHES
from .checkpoints import HASHES, CHUNK_TIMES
log = logging.getLogger(__name__)
@@ -138,7 +138,13 @@ class Headers:
def estimated_timestamp(self, height):
if height <= 0:
return
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
if height >= max(self.checkpoints.keys()):
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
chunk_index = min(height // 1000, len(CHUNK_TIMES) - 1)
offset = height - chunk_index * 1000
chunk_start_time = CHUNK_TIMES[chunk_index]
chunk_end_time = CHUNK_TIMES[chunk_index + 1]
return int(offset * ((chunk_end_time - chunk_start_time) / 1000) + chunk_start_time)
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
+1 -1
View File
@@ -929,7 +929,7 @@ class Ledger(metaclass=LedgerRegistry):
item = {
'txid': tx.id,
'timestamp': ts,
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'date': datetime.utcfromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
'claim_info': [],
'update_info': [],
+26 -10
View File
@@ -545,11 +545,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
if attr == 'tag':
any_queries[f'#_any_{attr}'] = f"""
(claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
"""
else:
any_queries[f'#_any_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
@@ -596,11 +604,19 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
if attr == 'tag':
constraints[f'#_not_{attr}'] = f"""
(claim.claim_type != {CLAIM_TYPES['repost']}
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) AND
(claim.claim_type == {CLAIM_TYPES['repost']} AND
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
"""
else:
constraints[f'#_not_{attr}'] = f"""
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
)
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
+8
View File
@@ -1,4 +1,5 @@
import asyncio
import textwrap
import os
from lbry.extras.cli import ensure_directory_exists
@@ -24,11 +25,18 @@ async def main():
return
print(f"Headers file at {headers.height}, checkpointing up to {target}."
f"Current checkpoint at {current_checkpoint_tip}.")
timestamps = []
with open(outpath, 'w') as outfile:
print('HASHES = {', file=outfile)
for height in range(0, target, 1000):
timestamps.append((await headers.get(height))['timestamp'])
print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile)
print('}', file=outfile)
print('CHUNK_TIMES = [', file=outfile)
for line in textwrap.wrap(f'{", ".join(map(str, timestamps))}', 120, initial_indent=' '*4, subsequent_indent=' '*4):
print(line, file=outfile)
print(']', file=outfile)
if __name__ == "__main__":
@@ -503,22 +503,35 @@ class TransactionOutputCommands(ClaimTestCase):
address2 = await self.daemon.jsonrpc_address_unused(wallet_id=wallet2.id)
await self.channel_create('@kept-channel')
await self.channel_create('@sent-channel', claim_address=address2)
await self.wallet_send('2.9', address2)
# all txos on second wallet
received_channel, = await self.txo_list(wallet_id=wallet2.id, is_my_input_or_output=True)
received_payment, received_channel = await self.txo_list(
wallet_id=wallet2.id, is_my_input_or_output=True)
self.assertEqual('1.0', received_channel['amount'])
self.assertFalse(received_channel['is_my_input'])
self.assertTrue(received_channel['is_my_output'])
self.assertFalse(received_channel['is_internal_transfer'])
self.assertEqual('2.9', received_payment['amount'])
self.assertFalse(received_payment['is_my_input'])
self.assertTrue(received_payment['is_my_output'])
self.assertFalse(received_payment['is_internal_transfer'])
# all txos on default wallet
r = await self.txo_list(is_my_input_or_output=True)
self.assertEqual(
['1.0', '7.947786', '1.0', '8.973893', '10.0'],
['2.9', '5.047662', '1.0', '7.947786', '1.0', '8.973893', '10.0'],
[t['amount'] for t in r]
)
sent_channel, change2, kept_channel, change1, initial_funds = r
sent_payment, change3, sent_channel, change2, kept_channel, change1, initial_funds = r
self.assertTrue(sent_payment['is_my_input'])
self.assertFalse(sent_payment['is_my_output'])
self.assertFalse(sent_payment['is_internal_transfer'])
self.assertTrue(change3['is_my_input'])
self.assertTrue(change3['is_my_output'])
self.assertTrue(change3['is_internal_transfer'])
self.assertTrue(sent_channel['is_my_input'])
self.assertFalse(sent_channel['is_my_output'])
@@ -540,31 +553,31 @@ class TransactionOutputCommands(ClaimTestCase):
# my stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, exclude_internal_transfers=True)
self.assertEqual([sent_channel, kept_channel, initial_funds], r)
self.assertEqual([sent_payment, sent_channel, kept_channel, initial_funds], r)
# my unspent stuff and stuff i sent excluding "change"
r = await self.txo_list(is_my_input_or_output=True, is_not_spent=True, exclude_internal_transfers=True)
self.assertEqual([sent_channel, kept_channel], r)
self.assertEqual([sent_payment, sent_channel, kept_channel], r)
# only "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other")
self.assertEqual([change2, change1], r)
self.assertEqual([change3, change2, change1], r)
# only unspent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_not_spent=True)
self.assertEqual([change2], r)
self.assertEqual([change3], r)
# only spent "change"
r = await self.txo_list(is_my_input=True, is_my_output=True, type="other", is_spent=True)
self.assertEqual([change1], r)
self.assertEqual([change2, change1], r)
# all my unspent stuff
r = await self.txo_list(is_my_output=True, is_not_spent=True)
self.assertEqual([change2, kept_channel], r)
self.assertEqual([change3, kept_channel], r)
# stuff i sent
r = await self.txo_list(is_not_my_output=True)
self.assertEqual([sent_channel], r)
self.assertEqual([sent_payment, sent_channel], r)
async def test_txo_plot(self):
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
@@ -21,6 +21,11 @@ class FileCommands(CommandTestCase):
self.assertEqual(file1['claim_name'], 'foo')
self.assertEqual(file2['claim_name'], 'foo2')
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=[file1['claim_id'], file2['claim_id']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(claim_id=file1['claim_id']), 1)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=[file1['outpoint'], file2['outpoint']]), 2)
self.assertItemCount(await self.daemon.jsonrpc_file_list(outpoint=file1['outpoint']), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo')
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
@@ -106,11 +106,16 @@ class MockedCommentServer:
return False
def hide_comments(self, pieces: list):
comments_hidden = []
hidden = []
for p in pieces:
if self.hide_comment(**p):
comments_hidden.append(p['comment_id'])
return {'hidden': comments_hidden}
hidden.append(p['comment_id'])
comment_ids = {c['comment_id'] for c in pieces}
return {
'hidden': hidden,
'visible': list(comment_ids - set(hidden))
}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
@@ -138,12 +143,19 @@ class MockedCommentServer:
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[self.get_comment_id(comment_id)]
return {
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
'channel_id': comment['channel_id'],
'channel_name': comment['channel_name'],
}
def get_comments_by_id(self, comment_ids: list):
return [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
return {
'page': 1,
'page_size': len(comment_ids),
'total_pages': 1,
'items': comments,
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
}
methods = {
'get_claim_comments': get_claim_comments,
+2
View File
@@ -46,7 +46,9 @@ class TestStreamAssembler(AsyncioTestCase):
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
self.assertEqual(0, self.stream.reflector_progress)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertEqual(100, self.stream.reflector_progress)
self.assertSetEqual(
set(sent),
set(map(lambda b: b.blob_hash,
+9
View File
@@ -153,6 +153,15 @@ class TestHeaders(AsyncioTestCase):
self.assertIsNone(headers.estimated_timestamp(0))
self.assertIsNotNone(headers.estimated_timestamp(1))
def test_estimate_from_checkpoints(self):
headers = _Headers(':memory:') # here we use the real version, with checkpoints
# check estimations matches real timestamps (LBRY, main net) by at most 20 minutes
self.assertAlmostEqual(headers.estimated_timestamp(101010), 1482899706, delta=60 * 20)
self.assertAlmostEqual(headers.estimated_timestamp(505050), 1547865484, delta=60 * 20)
self.assertAlmostEqual(headers.estimated_timestamp(707070), 1580258295, delta=60 * 20)
# tip uses old algorithm as the range doesnt exist
self.assertIsNotNone(headers.estimated_timestamp(max(headers.checkpoints.keys())))
async def test_misalignment_triggers_repair_on_open(self):
headers = Headers(':memory:')
headers.io.seek(0)