Compare commits

..
Author SHA1 Message Date
Jack Robison 5dc2dadb03 clarify second server.version isn't sent 2020-01-26 12:28:44 -05:00
Jack Robison 1af58c83f8 improve test_pick_fastest 2020-01-26 12:27:24 -05:00
Jack Robison a10ae70773 fix cache_query race condition 2020-01-26 12:26:46 -05:00
Alex Grintsvayg 65d9dca917 Merge branch 'docker'
* docker:
  fix gitlab ci
  add min_version and method name to metrics
  include docker tag in build info
  build_type.py -> build_info.py
  add docker build info to wallet server image
2020-01-23 13:13:33 -05:00
Alex Grintsvayg 2232b08351 fix gitlab ci 2020-01-23 13:13:20 -05:00
Alex Grintsvayg 0732ab6ad5 add min_version and method name to metrics 2020-01-23 13:13:20 -05:00
Alex Grintsvayg fb568768c5 include docker tag in build info 2020-01-23 13:13:19 -05:00
Alex Grintsvayg 7fd56e0add build_type.py -> build_info.py 2020-01-23 13:13:19 -05:00
Alex Grintsvayg 38a3f8cf4c add docker build info to wallet server image 2020-01-23 13:13:19 -05:00
Lex Berezhny 8d8a5b36b6 Merge pull request #2737 from lbryio/fix_repost_resolve_bug
fix bug where repost claims were not being fully resolved
2020-01-22 23:32:55 -05:00
Lex Berezhny a70cc7beb1 fix bug where repost claims were not being fully resolved 2020-01-22 22:59:05 -05:00
Alex Grintsvayg c5421699b1 add prometheus port to docker-compose.yml 2020-01-22 20:08:29 -05:00
Alex Grintsvayg d9f809864f include version and build info in prometheus output 2020-01-22 19:13:58 -05:00
Lex Berezhny bb23f509d7 rename BLOCKING_CHANNELS_IDS/FILTERING_CHANNELS_IDS to remove double S 2020-01-22 11:02:01 -05:00
Alex Grin 254e184677 Merge pull request #2734 from lbryio/prom2
Add prometheus metrics collection to client and server (take two)
2020-01-22 10:28:54 -05:00
Alex Grintsvayg 6c07141abd Add prometheus metrics collection to client and server 2020-01-22 10:28:17 -05:00
19 changed files with 172 additions and 52 deletions

No files matched your search

+2 -2
View File
@@ -72,8 +72,8 @@ test:json-api:
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller
- python3.7 docker/set_build.py
- pip install -e .
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
- chmod +x dist/lbrynet
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
@@ -129,8 +129,8 @@ build:windows:
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.5
- python docker/set_build.py
- pip install -e .
- python docker/set_build.py # must come after lbry is installed because it imports lbry
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
+5
View File
@@ -4,6 +4,10 @@ ARG user=lbry
ARG db_dir=/database
ARG projects_dir=/home/$user
ARG DOCKER_TAG
ARG DOCKER_COMMIT=docker
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
RUN apt-get update && \
apt-get -y --no-install-recommends install \
wget \
@@ -29,6 +33,7 @@ WORKDIR $projects_dir
RUN pip install uvloop
RUN make install
RUN python3 docker/set_build.py
RUN rm ~/.cache -rf
# entry point
+1
View File
@@ -26,6 +26,7 @@ services:
ports:
- "50001:50001" # rpc port
- "50005:50005" # websocket port
- "2112:2112" # prometheus
volumes:
- "wallet_server:/database"
environment:
+7
View File
@@ -0,0 +1,7 @@
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes
echo "docker build dir: $(pwd)"
docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .
+23 -20
View File
@@ -1,38 +1,41 @@
"""Set the build version to be 'qa', 'rc', 'release'"""
import sys
import os
import re
import logging
import lbry.build_info as build_info_mod
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
def get_build_type(ci_tag=None):
if not ci_tag:
return "qa"
log.debug("getting build type for tag: \"%s\"", ci_tag)
if re.match(r'v\d+\.\d+\.\d+rc\d+$', ci_tag):
return 'rc'
elif re.match(r'v\d+\.\d+\.\d+$', ci_tag):
return 'release'
return 'qa'
def _check_and_set(d: dict, key: str, value: str):
try:
d[key]
except KeyError:
raise Exception(f"{key} var does not exist in {build_info_mod.__file__}")
d[key] = value
def main():
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py')
log.debug("configuring build type file: %s", build_type_path)
commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT'))
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
if commit_hash is None:
raise ValueError("Commit hash not found in env vars")
commit_hash = commit_hash[:6]
build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG')))
log.debug("setting build type=%s, build commit=%s", build_type, commit_hash)
with open(build_type_path, 'w') as f:
f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{commit_hash}\"\n")
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
docker_tag = os.getenv('DOCKER_TAG')
if docker_tag:
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
_check_and_set(build_info, "BUILD", "docker")
else:
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
with open(build_info_mod.__file__, 'w') as f:
f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n")
if __name__ == '__main__':
+2 -1
View File
@@ -1,3 +1,4 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
COMMIT_HASH = "none"
DOCKER_TAG = "none"
+1
View File
@@ -502,6 +502,7 @@ class Config(CLIConfig):
)
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
# routing table
+28 -1
View File
@@ -17,6 +17,7 @@ from functools import wraps, partial
import ecdsa
import base58
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
from google.protobuf.message import DecodeError
from lbry.wallet import (
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
@@ -319,6 +320,10 @@ class Daemon(metaclass=JSONRPCServerType):
streaming_app.router.add_get('/stream/{sd_hash}', self.handle_stream_range_request)
self.streaming_runner = web.AppRunner(streaming_app)
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.metrics_runner = web.AppRunner(prom_app)
@property
def dht_node(self) -> typing.Optional['Node']:
return self.component_manager.get_component(DHT_COMPONENT)
@@ -446,6 +451,7 @@ class Daemon(metaclass=JSONRPCServerType):
await self.analytics_manager.send_server_startup()
await self.rpc_runner.setup()
await self.streaming_runner.setup()
await self.metrics_runner.setup()
try:
rpc_site = web.TCPSite(self.rpc_runner, self.conf.api_host, self.conf.api_port, shutdown_timeout=.5)
@@ -467,6 +473,16 @@ class Daemon(metaclass=JSONRPCServerType):
await self.analytics_manager.send_server_startup_error(str(e))
raise SystemExit()
if self.conf.prometheus_port:
try:
prom_site = web.TCPSite(self.metrics_runner, "0.0.0.0", self.conf.prometheus_port, shutdown_timeout=.5)
await prom_site.start()
log.info('metrics server listening on TCP %s:%i', *prom_site._server.sockets[0].getsockname()[:2])
except OSError as e:
log.error('metrics server failed to bind TCP :%i', self.conf.prometheus_port)
await self.analytics_manager.send_server_startup_error(str(e))
raise SystemExit()
try:
await self.initialize()
except asyncio.CancelledError:
@@ -498,6 +514,7 @@ class Daemon(metaclass=JSONRPCServerType):
log.info("stopped api components")
await self.rpc_runner.cleanup()
await self.streaming_runner.cleanup()
await self.metrics_runner.cleanup()
log.info("stopped api server")
if self.analytics_manager.is_started:
self.analytics_manager.stop()
@@ -527,6 +544,16 @@ class Daemon(metaclass=JSONRPCServerType):
content_type='application/json'
)
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
log.exception('could not generate prometheus data')
raise
async def handle_stream_get_request(self, request: web.Request):
if not self.conf.streaming_get:
log.warning("streaming_get is disabled, rejecting request")
@@ -595,7 +622,7 @@ class Daemon(metaclass=JSONRPCServerType):
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
# TODO: also delete EMPTY_PARAMS then
_args, _kwargs = (), args[0]
elif isinstance(args, list) and len(args) == 2 and\
elif isinstance(args, list) and len(args) == 2 and \
isinstance(args[0], list) and isinstance(args[1], dict):
_args, _kwargs = args
else:
+2 -2
View File
@@ -2,7 +2,7 @@ import platform
import os
import logging.handlers
from lbry import build_type, __version__ as lbrynet_version
from lbry import build_info, __version__ as lbrynet_version
log = logging.getLogger(__name__)
@@ -19,7 +19,7 @@ def get_platform() -> dict:
"os_system": os_system,
"lbrynet_version": lbrynet_version,
"version": lbrynet_version,
"build": build_type.BUILD, # CI server sets this during build step
"build": build_info.BUILD, # CI server sets this during build step
}
if d["os_system"] == "Linux":
import distro # pylint: disable=import-outside-toplevel
+1 -1
View File
@@ -94,7 +94,7 @@ class ClientSession(BaseClientSession):
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
elif (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.ensure_server_version()
retry_delay = default_delay
except RPCError as e:
+1 -1
View File
@@ -749,7 +749,7 @@ class LBRYBlockProcessor(BlockProcessor):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(20):
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
self.timer.show(height=height)
return undo
+15 -14
View File
@@ -233,19 +233,20 @@ class SQLDB:
self.filtered_channels.update(self.blocked_channels)
def update_claims_from_channel_hashes(self, shared_streams, shared_channels, channel_hashes):
sql = query(
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
'claim.reposted_claim_hash__is_not_null': 1,
'claim.channel_hash__in': channel_hashes
}
)
streams, channels = {}, {}
for blocked_claim in self.execute(*sql):
if blocked_claim.claim_type == CLAIM_TYPES['stream']:
streams[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
elif blocked_claim.claim_type == CLAIM_TYPES['channel']:
channels[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
if channel_hashes:
sql = query(
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
'claim.reposted_claim_hash__is_not_null': 1,
'claim.channel_hash__in': channel_hashes
}
)
for blocked_claim in self.execute(*sql):
if blocked_claim.claim_type == CLAIM_TYPES['stream']:
streams[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
elif blocked_claim.claim_type == CLAIM_TYPES['channel']:
channels[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
shared_streams.clear()
shared_streams.update(streams)
shared_channels.clear()
@@ -847,8 +848,8 @@ class LBRYLevelDB(LevelDB):
# space separated list of channel URIs used for filtering bad content
self.sql = SQLDB(
self, path,
self.env.default('BLOCKING_CHANNELS_IDS', '').split(' '),
self.env.default('FILTERING_CHANNELS_IDS', '').split(' '),
self.env.default('BLOCKING_CHANNEL_IDS', '').split(' '),
self.env.default('FILTERING_CHANNEL_IDS', '').split(' '),
)
def close(self):
+1
View File
@@ -59,6 +59,7 @@ class Env:
self.ssl_certfile = self.required('SSL_CERTFILE')
self.ssl_keyfile = self.required('SSL_KEYFILE')
self.rpc_port = self.integer('RPC_PORT', 8000)
self.prometheus_port = self.integer('PROMETHEUS_PORT', 0)
self.max_subscriptions = self.integer('MAX_SUBSCRIPTIONS', 10000)
self.banner_file = self.default('BANNER_FILE', None)
self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file)
+46
View File
@@ -0,0 +1,46 @@
from aiohttp import web
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest
from lbry import __version__ as version
from lbry.build_info import BUILD, COMMIT_HASH
from lbry.wallet.server import util
import lbry.wallet.server.version as wallet_server_version
NAMESPACE = "wallet_server"
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
VERSION_INFO.info({
'build': BUILD,
"commit": COMMIT_HASH,
'version': version,
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
})
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE, labelnames=("method",))
class PrometheusServer:
def __init__(self):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.runner = None
async def start(self, port: int):
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
await self.runner.setup()
metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
self.logger.exception('could not generate prometheus data')
raise
async def stop(self):
await self.runner.cleanup()
+12
View File
@@ -2,9 +2,11 @@ import signal
import logging
import asyncio
from concurrent.futures.thread import ThreadPoolExecutor
import typing
import lbry
from lbry.wallet.server.mempool import MemPool, MemPoolAPI
from lbry.wallet.server.prometheus import PrometheusServer
class Notifications:
@@ -74,6 +76,7 @@ class Server:
self.daemon = daemon = env.coin.DAEMON(env.coin, env.daemon_url)
self.db = db = env.coin.DB(env)
self.bp = bp = env.coin.BLOCK_PROCESSOR(env, db, daemon, notifications)
self.prometheus_server: typing.Optional[PrometheusServer] = None
# Set notifications up to implement the MemPoolAPI
notifications.height = daemon.height
@@ -107,11 +110,15 @@ class Server:
await self.db.populate_header_merkle_cache()
await _start_cancellable(self.mempool.keep_synchronized)
await _start_cancellable(self.session_mgr.serve, self.notifications)
await self.start_prometheus()
async def stop(self):
for task in reversed(self.cancellable_tasks):
task.cancel()
await asyncio.wait(self.cancellable_tasks)
if self.prometheus_server:
await self.prometheus_server.stop()
self.prometheus_server = None
self.shutdown_event.set()
await self.daemon.close()
@@ -132,3 +139,8 @@ class Server:
finally:
loop.run_until_complete(self.stop())
executor.shutdown(True)
async def start_prometheus(self):
if not self.prometheus_server and self.env.prometheus_port:
self.prometheus_server = PrometheusServer()
await self.prometheus_server.start(self.env.prometheus_port)
+8 -8
View File
@@ -27,6 +27,8 @@ from lbry.wallet.server.db.writer import LBRYLevelDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
from lbry.wallet.server.prometheus import REQUESTS_COUNT
import lbry.wallet.server.version as VERSION
from lbry.wallet.rpc import (
RPCSession, JSONRPCAutoDetect, JSONRPCConnection,
@@ -707,6 +709,7 @@ class SessionBase(RPCSession):
"""Handle an incoming request. ElectrumX doesn't receive
notifications from client sessions.
"""
REQUESTS_COUNT.labels(method=request.method).inc()
if isinstance(request, Request):
handler = self.request_handlers.get(request.method)
else:
@@ -776,8 +779,8 @@ class LBRYSessionManager(SessionManager):
class LBRYElectrumX(SessionBase):
"""A TCP server that handles incoming Electrum connections."""
PROTOCOL_MIN = (0, 54, 0)
PROTOCOL_MAX = (0, 99, 0)
PROTOCOL_MIN = VERSION.PROTOCOL_MIN
PROTOCOL_MAX = VERSION.PROTOCOL_MAX
max_errors = math.inf # don't disconnect people for errors! let them happen...
session_mgr: LBRYSessionManager
version = lbry.__version__
@@ -916,12 +919,9 @@ class LBRYElectrumX(SessionBase):
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
metrics.cache_response()
return cache_item.result
if cache_key not in cache:
cache[cache_key] = ResultCacheItem()
cache_item = cache[cache_key]
async with cache_item.lock:
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
+3
View File
@@ -0,0 +1,3 @@
# need this to avoid circular import
PROTOCOL_MIN = (0, 54, 0)
PROTOCOL_MAX = (0, 99, 0)
+1
View File
@@ -44,6 +44,7 @@ setup(
'cryptography==2.5',
'protobuf==3.6.1',
'msgpack==0.6.1',
'prometheus_client==0.7.1',
'ecdsa==0.13.3',
'pyyaml==4.2b1',
'docopt==0.6.2',
+13 -2
View File
@@ -8,6 +8,7 @@ from unittest.mock import Mock
from lbry.wallet.network import Network
from lbry.wallet.orchstr8.node import SPVNode
from lbry.wallet.rpc import RPCSession
from lbry.wallet.rpc import RPCError
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
@@ -140,12 +141,16 @@ class ReconnectTests(IntegrationTestCase):
class ServerPickingTestCase(AsyncioTestCase):
async def _make_fake_server(self, latency=1.0, port=1):
async def _make_fake_server(self, latency=1.0, port=1, raise_version_rpc_error=False, return_versions=None):
# local fake server with artificial latency
class FakeSession(RPCSession):
async def handle_request(self, request):
await asyncio.sleep(latency)
if request.method == 'server.version':
if raise_version_rpc_error:
raise RPCError(1, 'derp')
if return_versions:
return return_versions
return tuple(request.args)
return {'height': 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
@@ -155,7 +160,10 @@ class ServerPickingTestCase(AsyncioTestCase):
async def _make_bad_server(self, port=42420):
async def echo(reader, writer):
while True:
writer.write(await reader.read())
try:
writer.write(await asyncio.wait_for(reader.read(), 1))
except asyncio.TimeoutError:
pass
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
self.addCleanup(server.close)
return '127.0.0.1', port
@@ -169,6 +177,9 @@ class ServerPickingTestCase(AsyncioTestCase):
('example.that.doesnt.resolve', 9000),
await self._make_fake_server(latency=1.0, port=1340),
await self._make_fake_server(latency=0.1, port=1337),
await self._make_fake_server(latency=0, port=1338, raise_version_rpc_error=True),
await self._make_fake_server(latency=0, port=1341, return_versions=('0.0.1', '2.0')),
await self._make_fake_server(latency=0, port=1342, return_versions=('0.1', '2.0')),
await self._make_fake_server(latency=0.4, port=1339),
],
'connect_timeout': 3