Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
524961979d | ||
|
|
65d9dca917 | ||
|
|
2232b08351 | ||
|
|
0732ab6ad5 | ||
|
|
fb568768c5 | ||
|
|
7fd56e0add | ||
|
|
38a3f8cf4c | ||
|
|
8d8a5b36b6 | ||
|
|
a70cc7beb1 | ||
|
|
c5421699b1 | ||
|
|
d9f809864f | ||
|
|
bb23f509d7 | ||
|
|
254e184677 | ||
|
|
6c07141abd |
No files matched your search
+2
-2
@@ -72,8 +72,8 @@ test:json-api:
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- python3.7 docker/set_build.py
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
@@ -129,8 +129,8 @@ build:windows:
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- python docker/set_build.py
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
|
||||
@@ -4,6 +4,10 @@ ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
@@ -29,6 +33,7 @@ WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
|
||||
@@ -26,6 +26,7 @@ services:
|
||||
ports:
|
||||
- "50001:50001" # rpc port
|
||||
- "50005:50005" # websocket port
|
||||
- "2112:2112" # prometheus
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes
|
||||
echo "docker build dir: $(pwd)"
|
||||
|
||||
docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .
|
||||
+23
-20
@@ -1,38 +1,41 @@
|
||||
"""Set the build version to be 'qa', 'rc', 'release'"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import lbry.build_info as build_info_mod
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def get_build_type(ci_tag=None):
|
||||
if not ci_tag:
|
||||
return "qa"
|
||||
log.debug("getting build type for tag: \"%s\"", ci_tag)
|
||||
if re.match(r'v\d+\.\d+\.\d+rc\d+$', ci_tag):
|
||||
return 'rc'
|
||||
elif re.match(r'v\d+\.\d+\.\d+$', ci_tag):
|
||||
return 'release'
|
||||
return 'qa'
|
||||
def _check_and_set(d: dict, key: str, value: str):
|
||||
try:
|
||||
d[key]
|
||||
except KeyError:
|
||||
raise Exception(f"{key} var does not exist in {build_info_mod.__file__}")
|
||||
d[key] = value
|
||||
|
||||
|
||||
def main():
|
||||
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py')
|
||||
log.debug("configuring build type file: %s", build_type_path)
|
||||
commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT'))
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
commit_hash = commit_hash[:6]
|
||||
build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG')))
|
||||
log.debug("setting build type=%s, build commit=%s", build_type, commit_hash)
|
||||
with open(build_type_path, 'w') as f:
|
||||
f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{commit_hash}\"\n")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
|
||||
docker_tag = os.getenv('DOCKER_TAG')
|
||||
if docker_tag:
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
BUILD_COMMIT = "source installation"
|
||||
COMMIT_HASH = "none"
|
||||
DOCKER_TAG = "none"
|
||||
@@ -502,6 +502,7 @@ class Config(CLIConfig):
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
|
||||
@@ -17,6 +17,7 @@ from functools import wraps, partial
|
||||
import ecdsa
|
||||
import base58
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
from google.protobuf.message import DecodeError
|
||||
from lbry.wallet import (
|
||||
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
|
||||
@@ -319,6 +320,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
streaming_app.router.add_get('/stream/{sd_hash}', self.handle_stream_range_request)
|
||||
self.streaming_runner = web.AppRunner(streaming_app)
|
||||
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.metrics_runner = web.AppRunner(prom_app)
|
||||
|
||||
@property
|
||||
def dht_node(self) -> typing.Optional['Node']:
|
||||
return self.component_manager.get_component(DHT_COMPONENT)
|
||||
@@ -446,6 +451,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await self.analytics_manager.send_server_startup()
|
||||
await self.rpc_runner.setup()
|
||||
await self.streaming_runner.setup()
|
||||
await self.metrics_runner.setup()
|
||||
|
||||
try:
|
||||
rpc_site = web.TCPSite(self.rpc_runner, self.conf.api_host, self.conf.api_port, shutdown_timeout=.5)
|
||||
@@ -467,6 +473,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await self.analytics_manager.send_server_startup_error(str(e))
|
||||
raise SystemExit()
|
||||
|
||||
if self.conf.prometheus_port:
|
||||
try:
|
||||
prom_site = web.TCPSite(self.metrics_runner, "0.0.0.0", self.conf.prometheus_port, shutdown_timeout=.5)
|
||||
await prom_site.start()
|
||||
log.info('metrics server listening on TCP %s:%i', *prom_site._server.sockets[0].getsockname()[:2])
|
||||
except OSError as e:
|
||||
log.error('metrics server failed to bind TCP :%i', self.conf.prometheus_port)
|
||||
await self.analytics_manager.send_server_startup_error(str(e))
|
||||
raise SystemExit()
|
||||
|
||||
try:
|
||||
await self.initialize()
|
||||
except asyncio.CancelledError:
|
||||
@@ -498,6 +514,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
log.info("stopped api components")
|
||||
await self.rpc_runner.cleanup()
|
||||
await self.streaming_runner.cleanup()
|
||||
await self.metrics_runner.cleanup()
|
||||
log.info("stopped api server")
|
||||
if self.analytics_manager.is_started:
|
||||
self.analytics_manager.stop()
|
||||
@@ -527,6 +544,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
content_type='application/json'
|
||||
)
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
log.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def handle_stream_get_request(self, request: web.Request):
|
||||
if not self.conf.streaming_get:
|
||||
log.warning("streaming_get is disabled, rejecting request")
|
||||
@@ -595,7 +622,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
|
||||
# TODO: also delete EMPTY_PARAMS then
|
||||
_args, _kwargs = (), args[0]
|
||||
elif isinstance(args, list) and len(args) == 2 and\
|
||||
elif isinstance(args, list) and len(args) == 2 and \
|
||||
isinstance(args[0], list) and isinstance(args[1], dict):
|
||||
_args, _kwargs = args
|
||||
else:
|
||||
|
||||
@@ -2,7 +2,7 @@ import platform
|
||||
import os
|
||||
import logging.handlers
|
||||
|
||||
from lbry import build_type, __version__ as lbrynet_version
|
||||
from lbry import build_info, __version__ as lbrynet_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,7 +19,7 @@ def get_platform() -> dict:
|
||||
"os_system": os_system,
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"version": lbrynet_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
"build": build_info.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if d["os_system"] == "Linux":
|
||||
import distro # pylint: disable=import-outside-toplevel
|
||||
|
||||
@@ -87,7 +87,7 @@ class ClientSession(BaseClientSession):
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
retry_delay = default_delay = 1.0
|
||||
retry_delay = default_delay = 300.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
@@ -107,7 +107,6 @@ class ClientSession(BaseClientSession):
|
||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
|
||||
@@ -749,7 +749,7 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
|
||||
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
||||
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(20):
|
||||
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
|
||||
self.timer.show(height=height)
|
||||
return undo
|
||||
|
||||
|
||||
@@ -233,19 +233,20 @@ class SQLDB:
|
||||
self.filtered_channels.update(self.blocked_channels)
|
||||
|
||||
def update_claims_from_channel_hashes(self, shared_streams, shared_channels, channel_hashes):
|
||||
sql = query(
|
||||
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
|
||||
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
|
||||
'claim.reposted_claim_hash__is_not_null': 1,
|
||||
'claim.channel_hash__in': channel_hashes
|
||||
}
|
||||
)
|
||||
streams, channels = {}, {}
|
||||
for blocked_claim in self.execute(*sql):
|
||||
if blocked_claim.claim_type == CLAIM_TYPES['stream']:
|
||||
streams[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
|
||||
elif blocked_claim.claim_type == CLAIM_TYPES['channel']:
|
||||
channels[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
|
||||
if channel_hashes:
|
||||
sql = query(
|
||||
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
|
||||
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
|
||||
'claim.reposted_claim_hash__is_not_null': 1,
|
||||
'claim.channel_hash__in': channel_hashes
|
||||
}
|
||||
)
|
||||
for blocked_claim in self.execute(*sql):
|
||||
if blocked_claim.claim_type == CLAIM_TYPES['stream']:
|
||||
streams[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
|
||||
elif blocked_claim.claim_type == CLAIM_TYPES['channel']:
|
||||
channels[blocked_claim.reposted_claim_hash] = blocked_claim.channel_hash
|
||||
shared_streams.clear()
|
||||
shared_streams.update(streams)
|
||||
shared_channels.clear()
|
||||
@@ -847,8 +848,8 @@ class LBRYLevelDB(LevelDB):
|
||||
# space separated list of channel URIs used for filtering bad content
|
||||
self.sql = SQLDB(
|
||||
self, path,
|
||||
self.env.default('BLOCKING_CHANNELS_IDS', '').split(' '),
|
||||
self.env.default('FILTERING_CHANNELS_IDS', '').split(' '),
|
||||
self.env.default('BLOCKING_CHANNEL_IDS', '').split(' '),
|
||||
self.env.default('FILTERING_CHANNEL_IDS', '').split(' '),
|
||||
)
|
||||
|
||||
def close(self):
|
||||
|
||||
@@ -59,6 +59,7 @@ class Env:
|
||||
self.ssl_certfile = self.required('SSL_CERTFILE')
|
||||
self.ssl_keyfile = self.required('SSL_KEYFILE')
|
||||
self.rpc_port = self.integer('RPC_PORT', 8000)
|
||||
self.prometheus_port = self.integer('PROMETHEUS_PORT', 0)
|
||||
self.max_subscriptions = self.integer('MAX_SUBSCRIPTIONS', 10000)
|
||||
self.banner_file = self.default('BANNER_FILE', None)
|
||||
self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from aiohttp import web
|
||||
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest
|
||||
from lbry import __version__ as version
|
||||
from lbry.build_info import BUILD, COMMIT_HASH
|
||||
from lbry.wallet.server import util
|
||||
import lbry.wallet.server.version as wallet_server_version
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
|
||||
VERSION_INFO.info({
|
||||
'build': BUILD,
|
||||
"commit": COMMIT_HASH,
|
||||
'version': version,
|
||||
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
|
||||
})
|
||||
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE, labelnames=("method",))
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self):
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.runner = None
|
||||
|
||||
async def start(self, port: int):
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
@@ -2,9 +2,11 @@ import signal
|
||||
import logging
|
||||
import asyncio
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
import typing
|
||||
|
||||
import lbry
|
||||
from lbry.wallet.server.mempool import MemPool, MemPoolAPI
|
||||
from lbry.wallet.server.prometheus import PrometheusServer
|
||||
|
||||
|
||||
class Notifications:
|
||||
@@ -74,6 +76,7 @@ class Server:
|
||||
self.daemon = daemon = env.coin.DAEMON(env.coin, env.daemon_url)
|
||||
self.db = db = env.coin.DB(env)
|
||||
self.bp = bp = env.coin.BLOCK_PROCESSOR(env, db, daemon, notifications)
|
||||
self.prometheus_server: typing.Optional[PrometheusServer] = None
|
||||
|
||||
# Set notifications up to implement the MemPoolAPI
|
||||
notifications.height = daemon.height
|
||||
@@ -107,11 +110,15 @@ class Server:
|
||||
await self.db.populate_header_merkle_cache()
|
||||
await _start_cancellable(self.mempool.keep_synchronized)
|
||||
await _start_cancellable(self.session_mgr.serve, self.notifications)
|
||||
await self.start_prometheus()
|
||||
|
||||
async def stop(self):
|
||||
for task in reversed(self.cancellable_tasks):
|
||||
task.cancel()
|
||||
await asyncio.wait(self.cancellable_tasks)
|
||||
if self.prometheus_server:
|
||||
await self.prometheus_server.stop()
|
||||
self.prometheus_server = None
|
||||
self.shutdown_event.set()
|
||||
await self.daemon.close()
|
||||
|
||||
@@ -132,3 +139,8 @@ class Server:
|
||||
finally:
|
||||
loop.run_until_complete(self.stop())
|
||||
executor.shutdown(True)
|
||||
|
||||
async def start_prometheus(self):
|
||||
if not self.prometheus_server and self.env.prometheus_port:
|
||||
self.prometheus_server = PrometheusServer()
|
||||
await self.prometheus_server.start(self.env.prometheus_port)
|
||||
@@ -27,6 +27,8 @@ from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
from lbry.wallet.server.prometheus import REQUESTS_COUNT
|
||||
import lbry.wallet.server.version as VERSION
|
||||
|
||||
from lbry.wallet.rpc import (
|
||||
RPCSession, JSONRPCAutoDetect, JSONRPCConnection,
|
||||
@@ -707,6 +709,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle an incoming request. ElectrumX doesn't receive
|
||||
notifications from client sessions.
|
||||
"""
|
||||
REQUESTS_COUNT.labels(method=request.method).inc()
|
||||
if isinstance(request, Request):
|
||||
handler = self.request_handlers.get(request.method)
|
||||
else:
|
||||
@@ -776,8 +779,8 @@ class LBRYSessionManager(SessionManager):
|
||||
class LBRYElectrumX(SessionBase):
|
||||
"""A TCP server that handles incoming Electrum connections."""
|
||||
|
||||
PROTOCOL_MIN = (0, 54, 0)
|
||||
PROTOCOL_MAX = (0, 99, 0)
|
||||
PROTOCOL_MIN = VERSION.PROTOCOL_MIN
|
||||
PROTOCOL_MAX = VERSION.PROTOCOL_MAX
|
||||
max_errors = math.inf # don't disconnect people for errors! let them happen...
|
||||
session_mgr: LBRYSessionManager
|
||||
version = lbry.__version__
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# need this to avoid circular import
|
||||
PROTOCOL_MIN = (0, 54, 0)
|
||||
PROTOCOL_MAX = (0, 99, 0)
|
||||
Reference in new issue
Block a user