Compare commits
85
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
265eb9da67 | ||
|
|
21c71bfac1 | ||
|
|
6baaed3581 | ||
|
|
152dbfd5d1 | ||
|
|
a56d14086b | ||
|
|
aee87693f8 | ||
|
|
976b4affd9 | ||
|
|
e222b6ad9c | ||
|
|
19b17374e8 | ||
|
|
43989122bb | ||
|
|
72712d6047 | ||
|
|
0b52d2cc15 | ||
|
|
8304102136 | ||
|
|
3381aefcfa | ||
|
|
279a365cb1 | ||
|
|
2c9e00da56 | ||
|
|
f7cae69704 | ||
|
|
b7d58bcdbc | ||
|
|
13a856b843 | ||
|
|
8da38985c3 | ||
|
|
60cf6c6b97 | ||
|
|
35c2b34564 | ||
|
|
ef2e048efc | ||
|
|
6b3261aa33 | ||
|
|
1849c02cb6 | ||
|
|
1ec74a89e2 | ||
|
|
c591792de9 | ||
|
|
3108543ae5 | ||
|
|
1eb221c743 | ||
|
|
bebf6bc2e7 | ||
|
|
9e91cc2138 | ||
|
|
c5b939cfb7 | ||
|
|
5bd411ca27 | ||
|
|
a533cda6f0 | ||
|
|
fe4b07b8ae | ||
|
|
f9f2ccd904 | ||
|
|
d9e87d7c32 | ||
|
|
a0092c0770 | ||
|
|
3100131125 | ||
|
|
988880cf83 | ||
|
|
c3fb9672c4 | ||
|
|
0a2d94e425 | ||
|
|
8d9073cd31 | ||
|
|
d075961ffa | ||
|
|
7a72409b61 | ||
|
|
34fc530fba | ||
|
|
f257ff2f97 | ||
|
|
7ad5822c5b | ||
|
|
9a8f9f0a94 | ||
|
|
6421cecafb | ||
|
|
be544d6d89 | ||
|
|
3c89ecafdd | ||
|
|
35ec4eec52 | ||
|
|
e47f737a2f | ||
|
|
ac671a065b | ||
|
|
74116cc550 | ||
|
|
406070a5c3 | ||
|
|
0ccafd5b53 | ||
|
|
940f517aa3 | ||
|
|
216e5f65ad | ||
|
|
a74685d66d | ||
|
|
b7791d2845 | ||
|
|
d151a82d78 | ||
|
|
8ce61fbd52 | ||
|
|
90c24aade3 | ||
|
|
6b3f787fee | ||
|
|
4ebe4ce1b7 | ||
|
|
8c79740ee8 | ||
|
|
59d027ca02 | ||
|
|
37a7345a90 | ||
|
|
c519d4651b | ||
|
|
9b3b609e40 | ||
|
|
6254f53716 | ||
|
|
f05dc46432 | ||
|
|
3de0982a4a | ||
|
|
c2184fb3bf | ||
|
|
919c09fcb0 | ||
|
|
1d9dbd40ec | ||
|
|
0cd953a6f3 | ||
|
|
4db2b72351 | ||
|
|
dd54fcbdbd | ||
|
|
3123cf7ac6 | ||
|
|
6b579dd4ce | ||
|
|
16dfaa3e27 | ||
|
|
d7842b9f84 |
No files matched your search
@@ -1,67 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
schedule:
|
||||
- cron: '21 0 * * 4'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'python' ]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
|
||||
# Learn more:
|
||||
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
+95
-24
@@ -1,5 +1,10 @@
|
||||
name: ci
|
||||
on: pull_request
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]+.[0-9]+.[0-9]+'
|
||||
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
@@ -10,18 +15,41 @@ jobs:
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,lint]
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
#- macos-latest
|
||||
#- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,test]
|
||||
- env:
|
||||
HOME: /tmp
|
||||
run: make test-unit-coverage
|
||||
@@ -38,15 +66,14 @@ jobs:
|
||||
steps:
|
||||
- name: Configure sysctl limits
|
||||
run: |
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
- name: Runs Elasticsearch
|
||||
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||
with:
|
||||
stack-version: 7.6.0
|
||||
stack-version: 7.6.0
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
@@ -55,16 +82,22 @@ jobs:
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./.tox
|
||||
key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
|
||||
restore-keys: txo-integration-${{ matrix.test }}-
|
||||
- run: pip install tox
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
name: "build / binary"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- ubuntu-16.04
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
@@ -73,22 +106,60 @@ jobs:
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
# https://stackoverflow.com/a/61693590
|
||||
# https://github.com/pypa/setuptools/issues/1963
|
||||
pip install --upgrade 'setuptools<45.0.0'
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ runner.os }}
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install pyinstaller==4.4
|
||||
- run: pip install -e .
|
||||
- if: startsWith(github.ref, 'refs/tags/v')
|
||||
run: python docker/set_build.py
|
||||
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(matrix.os, 'windows')
|
||||
- if: startsWith(runner.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pip install pywin32==301
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: lbrynet-${{ steps.os-name.outputs.lowercase }}
|
||||
path: dist/
|
||||
|
||||
release:
|
||||
name: "release"
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: ["build"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/download-artifact@v2
|
||||
- name: upload binaries
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
|
||||
run: |
|
||||
pip install githubrelease
|
||||
chmod +x lbrynet-macos/lbrynet
|
||||
chmod +x lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
|
||||
zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
|
||||
ls -lh
|
||||
githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
|
||||
githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
|
||||
lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
|
||||
githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
# This workflow integrates a collection of open source static analysis tools
|
||||
# with GitHub code scanning. For documentation, or to provide feedback, visit
|
||||
# https://github.com/github/ossar-action
|
||||
name: OSSAR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
schedule:
|
||||
- cron: '28 17 * * 1'
|
||||
|
||||
jobs:
|
||||
OSSAR-Scan:
|
||||
# OSSAR runs on windows-latest.
|
||||
# ubuntu-latest and macos-latest support coming soon
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Ensure a compatible version of dotnet is installed.
|
||||
# The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201.
|
||||
# A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action.
|
||||
# GitHub hosted runners already have a compatible version of dotnet installed and this step may be skipped.
|
||||
# For self-hosted runners, ensure dotnet version 3.1.201 or later is installed by including this action:
|
||||
# - name: Install .NET
|
||||
# uses: actions/setup-dotnet@v1
|
||||
# with:
|
||||
# dotnet-version: '3.1.x'
|
||||
|
||||
# Run open source static analysis tools
|
||||
- name: Run OSSAR
|
||||
uses: github/ossar-action@v1
|
||||
id: ossar
|
||||
|
||||
# Upload results to the Security tab
|
||||
- name: Upload OSSAR results
|
||||
uses: github/codeql-action/upload-sarif@v1
|
||||
with:
|
||||
sarif_file: ${{ steps.ossar.outputs.sarifFile }}
|
||||
@@ -0,0 +1,22 @@
|
||||
name: slack
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: "slack notification"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
|
||||
id: markdown
|
||||
with:
|
||||
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||
- uses: slackapi/slack-github-action@v1.14.0
|
||||
env:
|
||||
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||
with:
|
||||
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }}'
|
||||
|
||||
-220
@@ -1,220 +0,0 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
services:
|
||||
- name: docker.elastic.co/elasticsearch/elasticsearch:7.11.2
|
||||
alias: elasticsearch
|
||||
command: [ "bin/elasticsearch", "-Expack.security.enabled=false", "-Ediscovery.type=single-node" ]
|
||||
variables:
|
||||
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- assets
|
||||
- release
|
||||
|
||||
|
||||
.tagged:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
||||
when: on_success
|
||||
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- make lint
|
||||
|
||||
test:unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- ELASTIC_HOST=elasticsearch tox -e datanetwork --recreate
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- curl "http://elasticsearch:9200/_cat/health"
|
||||
- pip install tox-travis
|
||||
- ELASTIC_HOST=elasticsearch tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- ELASTIC_HOST=elasticsearch tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- ELASTIC_HOST=elasticsearch HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
- pip install lbry-libtorrent
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x64 python --version=3.7.9
|
||||
- choco install -y 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
|
||||
# s3 = upload asset to s3 (build.lbry.io)
|
||||
.s3:
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .s3
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
s3:mac:
|
||||
extends: .s3
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
s3:windows:
|
||||
extends: .s3
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
# github = upload assets to github when there's a tagged release
|
||||
.github:
|
||||
extends: .tagged
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
github:linux:
|
||||
extends: .github
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
github:mac:
|
||||
extends: .github
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
github:windows:
|
||||
extends: .github
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
publish:
|
||||
extends: .tagged
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
||||
- >
|
||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
||||
+104
-47
@@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
|
||||
at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
|
||||
If Python 3.8+ is used, the daemon will start but the RPC server
|
||||
may not accept messages, returning the following:
|
||||
```
|
||||
Could not connect to daemon. Are you sure it's running?
|
||||
```
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
These environment variables also need to be set:
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
```
|
||||
PYTHONUNBUFFERED=1
|
||||
EVENT_NOKQUEUE=1
|
||||
```
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python protobuf
|
||||
```
|
||||
@@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
|
||||
|
||||
On Ubuntu (we recommend 18.04 or 20.04), install the following:
|
||||
```
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
|
||||
for those Ubuntu distributions that no longer have it in their
|
||||
official repositories.
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
@@ -47,49 +59,68 @@ If you're running another Linux distro, install the equivalent of the above pack
|
||||
|
||||
### Linux/Mac
|
||||
|
||||
To install on Linux/Mac:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
$ git clone https://github.com/lbryio/lbry-sdk.git
|
||||
$ cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python3.7 -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
$ python3.7 -m venv lbry-venv
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
|
||||
$ python --version
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Install packages:
|
||||
$ make install
|
||||
Make sure you're on Python 3.7+ as default in the virtual environment:
|
||||
```bash
|
||||
python --version
|
||||
```
|
||||
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
$ make idea
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
```bash
|
||||
make idea
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside
|
||||
of the `lbry-venv` folder.
|
||||
```bash
|
||||
(lbry-venv) $ which lbrynet
|
||||
/opt/lbry-sdk/lbry-venv/bin/lbrynet
|
||||
```
|
||||
|
||||
To exit the virtual environment simply use the command `deactivate`.
|
||||
|
||||
### Windows
|
||||
|
||||
To install on Windows:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
> git clone https://github.com/lbryio/lbry-sdk.git
|
||||
> cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
> python -m venv lbry-venv
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
lbry-venv\Scripts\activate
|
||||
```
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
> lbry-venv\Scripts\activate
|
||||
|
||||
Install packages:
|
||||
> pip install -e .
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
### Elasticsearch
|
||||
@@ -100,21 +131,47 @@ The easiest way to start it is using docker with:
|
||||
```bash
|
||||
make elastic-docker
|
||||
```
|
||||
|
||||
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
|
||||
|
||||
|
||||
|
||||
To run the unit and integration tests from the repo directory:
|
||||
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
python -m unittest discover tests.integration
|
||||
```
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
python -m unittest discover tests.integration
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
```
|
||||
lbrynet start
|
||||
```
|
||||
|
||||
Whenever the code inside [lbry-sdk/lbry](./lbry)
|
||||
is modified we should run `make install` to recompile the `lbrynet`
|
||||
executable with the newest code.
|
||||
|
||||
## Development
|
||||
|
||||
When developing, remember to enter the environment,
|
||||
and if you wish start the server interactively.
|
||||
```bash
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
(lbry-venv) $ python lbry/extras/cli.py start
|
||||
```
|
||||
|
||||
Parameters can be passed in the same way.
|
||||
```bash
|
||||
(lbry-venv) $ python lbry/extras/cli.py wallet balance
|
||||
```
|
||||
|
||||
If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
|
||||
in this way, set up break points, and step through the code.
|
||||
```bash
|
||||
(lbry-venv) $ pip install ipdb
|
||||
|
||||
(lbry-venv) $ ipdb lbry/extras/cli.py
|
||||
```
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,7 +1,6 @@
|
||||
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||
|
||||
install:
|
||||
pip install lbry-libtorrent
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
|
||||
@@ -13,6 +13,7 @@ RUN apt-get update && \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
automake libtool \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG downloads_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
automake libtool \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $downloads_dir
|
||||
RUN chown -R $user:$user $downloads_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
VOLUME $downloads_dir
|
||||
COPY ./docker/webconf.yaml /webconf.yaml
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]
|
||||
@@ -0,0 +1,9 @@
|
||||
### How to run with docker-compose
|
||||
1. Edit config file and after that fix permissions with
|
||||
```
|
||||
sudo chown -R 999:999 webconf.yaml
|
||||
```
|
||||
2. Start SDK with
|
||||
```
|
||||
docker-compose up -d
|
||||
```
|
||||
@@ -0,0 +1,9 @@
|
||||
version: '3'
|
||||
services:
|
||||
websdk:
|
||||
image: vshyba/websdk
|
||||
ports:
|
||||
- '5279:5279'
|
||||
- '5280:5280'
|
||||
volumes:
|
||||
- ./webconf.yaml:/webconf.yaml
|
||||
+5
-3
@@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
|
||||
def main():
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
@@ -30,8 +30,10 @@ def main():
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
|
||||
_check_and_set(build_info, "BUILD", "release")
|
||||
else:
|
||||
_check_and_set(build_info, "BUILD", "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
allowed_origin: "*"
|
||||
max_key_fee: "0.0 USD"
|
||||
save_files: false
|
||||
save_blobs: false
|
||||
streaming_server: "0.0.0.0:5280"
|
||||
api: "0.0.0.0:5279"
|
||||
data_dir: /tmp
|
||||
download_dir: /tmp
|
||||
wallet_dir: /tmp
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.100.0"
|
||||
__version__ = "0.101.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+2
-2
@@ -156,9 +156,9 @@ class KademliaPeer:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
|
||||
@@ -452,7 +452,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
except ValueError as err:
|
||||
log.warning("error replying to %s: %s", address[0], str(err))
|
||||
return
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
|
||||
@@ -53,10 +53,11 @@ Code | Name | Message
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
423 | TooManyClaimSearchParameters | {key} cant be set for more than {limit} items.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
|
||||
+11
-3
@@ -215,10 +215,10 @@ class ResolveTimeoutError(WalletError):
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_hash):
|
||||
def __init__(self, url, censor_id):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
self.censor_id = censor_id
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
@@ -242,6 +242,14 @@ class IncompatibleWalletServerError(WalletError):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class TooManyClaimSearchParametersError(WalletError):
|
||||
|
||||
def __init__(self, key, limit):
|
||||
self.key = key
|
||||
self.limit = limit
|
||||
super().__init__(f"{key} cant be set for more than {limit} items.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
|
||||
@@ -190,6 +190,10 @@ def get_argument_parser():
|
||||
'--no-logging', dest='no_logging', action="store_true",
|
||||
help='Disable all logging of any kind.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--show-stack-trace', dest='show_stack_trace', action="store_true",
|
||||
help='Show full stack trace on error.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
|
||||
@@ -255,10 +259,28 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def make_stack_trace_supressing_record_factory(original_factory):
|
||||
def factory(*args, **kwargs):
|
||||
record = original_factory(*args, **kwargs)
|
||||
if record.exc_info:
|
||||
record.msg += f" ({record.exc_info[0].__name__}: {record.exc_info[1]})"
|
||||
record.exc_info = None
|
||||
record.exc_text = None
|
||||
return record
|
||||
return factory
|
||||
|
||||
|
||||
def warning_error_handler(loop, context):
|
||||
logging.warning("Uncaught exception: %s", context)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
elif not args.show_stack_trace:
|
||||
loop.set_exception_handler(warning_error_handler)
|
||||
logging.setLogRecordFactory(make_stack_trace_supressing_record_factory(logging.getLogRecordFactory()))
|
||||
if not args.no_logging:
|
||||
setup_logging(logging.getLogger(), args, conf)
|
||||
daemon = Daemon(conf)
|
||||
|
||||
@@ -32,15 +32,17 @@ TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _event_properties(analytics_id: str, event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'analytics_id': analytics_id
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, resolve_duration: float,
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: typing.Optional[int],
|
||||
tried_peers_count: typing.Optional[int], connection_failures_count: typing.Optional[int],
|
||||
@@ -52,6 +54,7 @@ def _download_properties(conf: Config, resolve_duration: float,
|
||||
error: typing.Optional[str] = None, error_msg: typing.Optional[str] = None,
|
||||
wallet_server: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
@@ -102,15 +105,17 @@ def _make_context(platform):
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
def __init__(self, conf: Config):
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.analytics_id = utils.generate_id().hex()
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
@@ -127,6 +132,7 @@ class AnalyticsManager:
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
@@ -188,7 +194,7 @@ class AnalyticsManager:
|
||||
error_msg: typing.Optional[str] = None,
|
||||
wallet_server: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, connection_failures_count, added_fixed_peers, fixed_peers_delay,
|
||||
sd_hash, sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error, error_msg,
|
||||
wallet_server
|
||||
@@ -221,7 +227,7 @@ class AnalyticsManager:
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': _event_properties(self.analytics_id, properties),
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
@@ -481,6 +481,10 @@ class UPnPComponent(Component):
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
self.external_ip = external_ip
|
||||
dht_component = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_component:
|
||||
dht_node = dht_component.component
|
||||
dht_node.protocol.external_ip = external_ip
|
||||
# assert self.external_ip is not None # TODO: handle going/starting offline
|
||||
|
||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||
|
||||
@@ -325,7 +325,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
self._video_file_analyzer = VideoFileAnalyzer(conf)
|
||||
self._node_id = None
|
||||
self._installation_id = None
|
||||
self.analytics_manager = analytics.AnalyticsManager(conf)
|
||||
self.session_id = base58.b58encode(utils.generate_id()).decode()
|
||||
self.analytics_manager = analytics.AnalyticsManager(conf, self.installation_id, self.session_id)
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
conf, analytics_manager=self.analytics_manager,
|
||||
skip_components=conf.components_to_skip or []
|
||||
@@ -2478,6 +2479,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None))
|
||||
if "claim_ids" in kwargs and not kwargs["claim_ids"]:
|
||||
kwargs.pop("claim_ids")
|
||||
if {'claim_id', 'claim_ids'}.issubset(kwargs):
|
||||
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
|
||||
if kwargs.pop('valid_channel_signature', False):
|
||||
|
||||
@@ -186,7 +186,7 @@ class UPbitBTCFeed(MarketFeed):
|
||||
params = {"markets": "BTC-LBC"}
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response[0]['trade_price'])
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ class Censor:
|
||||
|
||||
def censor(self, row) -> bool:
|
||||
if self.is_censored(row):
|
||||
censoring_channel_hash = row['censoring_channel_hash']
|
||||
censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
|
||||
self.censored.setdefault(censoring_channel_hash, set())
|
||||
self.censored[censoring_channel_hash].add(row['tx_hash'])
|
||||
return True
|
||||
@@ -179,7 +179,7 @@ class Outputs:
|
||||
txo_message.error.code = ErrorMessage.NOT_FOUND
|
||||
elif isinstance(txo, ResolveCensoredError):
|
||||
txo_message.error.code = ErrorMessage.BLOCKED
|
||||
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(txo.censor_hash))
|
||||
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(bytes.fromhex(txo.censor_id)[::-1]))
|
||||
return
|
||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
|
||||
@@ -96,7 +96,8 @@ class AddressManager:
|
||||
return [r['address'] for r in records]
|
||||
|
||||
async def get_or_create_usable_address(self) -> str:
|
||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||
async with self.address_generator_lock:
|
||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||
if addresses:
|
||||
return random.choice(addresses)
|
||||
addresses = await self.ensure_address_gap()
|
||||
|
||||
@@ -750,7 +750,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id:
|
||||
if local_height >= height:
|
||||
if local_height >= height or (local_height == 0 and height > local_height):
|
||||
return True
|
||||
log.warning(
|
||||
"local history has higher height than remote for %s (%i vs %i)", txid,
|
||||
|
||||
@@ -436,7 +436,8 @@ class RPCSession(SessionBase):
|
||||
except CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
self.logger.exception(f'exception handling {request}')
|
||||
reqstr = str(request)
|
||||
self.logger.exception(f'exception handling {reqstr[:16_000]}')
|
||||
result = RPCError(JSONRPC.INTERNAL_ERROR,
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
|
||||
@@ -35,27 +35,44 @@ INDEX_DEFAULT_SETTINGS = {
|
||||
}
|
||||
}
|
||||
}
|
||||
FIELDS = {'is_controlling', 'last_take_over_height', 'claim_id', 'claim_name', 'normalized', 'tx_position', 'amount',
|
||||
'timestamp', 'creation_timestamp', 'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'release_time', 'short_url', 'canonical_url', 'title', 'author', 'description', 'claim_type', 'reposted',
|
||||
'stream_type', 'media_type', 'fee_amount', 'fee_currency', 'duration', 'reposted_claim_hash', 'censor_type',
|
||||
'claims_in_channel', 'channel_join', 'signature_valid', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'channel_id', 'tx_id', 'tx_nout',
|
||||
'signature', 'signature_digest', 'public_key_bytes', 'public_key_hash', 'public_key_id', '_id', 'tags',
|
||||
'reposted_claim_id'}
|
||||
TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id',
|
||||
'media_type', 'normalized', 'public_key_bytes', 'public_key_hash', 'short_url', 'signature',
|
||||
|
||||
FIELDS = {
|
||||
'_id',
|
||||
'claim_id', 'claim_type', 'claim_name', 'normalized_name',
|
||||
'tx_id', 'tx_nout', 'tx_position',
|
||||
'short_url', 'canonical_url',
|
||||
'is_controlling', 'last_take_over_height',
|
||||
'public_key_bytes', 'public_key_id', 'claims_in_channel', 'channel_join_height',
|
||||
'channel_id', 'signature', 'signature_digest', 'is_signature_valid',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'fee_amount', 'fee_currency',
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'stream_type', 'media_type', 'censor_type',
|
||||
'title', 'author', 'description',
|
||||
'timestamp', 'creation_timestamp',
|
||||
'duration', 'release_time',
|
||||
'tags', 'languages', 'has_source', 'reposted_claim_type',
|
||||
'reposted_claim_id', 'repost_count',
|
||||
'trending_group', 'trending_mixed', 'trending_local', 'trending_global',
|
||||
}
|
||||
|
||||
TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id', 'censoring_channel_id',
|
||||
'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature',
|
||||
'signature_digest', 'stream_type', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', 'tags'}
|
||||
|
||||
RANGE_FIELDS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
|
||||
'tx_position', 'channel_join', 'repost_count', 'limit_claims_per_channel',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed', 'censor_type',
|
||||
'trending_local', 'trending_global',
|
||||
}
|
||||
|
||||
ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS
|
||||
|
||||
REPLACEMENTS = {
|
||||
'name': 'normalized',
|
||||
'name': 'normalized_name',
|
||||
'txid': 'tx_id',
|
||||
'claim_hash': '_id'
|
||||
'reposted': 'repost_count'
|
||||
}
|
||||
@@ -10,14 +10,14 @@ from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
|
||||
from elasticsearch.helpers import async_streaming_bulk
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
|
||||
from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES
|
||||
from lbry.wallet.server.db.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, \
|
||||
RANGE_FIELDS
|
||||
RANGE_FIELDS, ALL_FIELDS
|
||||
from lbry.wallet.server.util import class_logger
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ class SearchIndex:
|
||||
update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
|
||||
key = 'channel_id' if channels else 'claim_id'
|
||||
update['script'] = {
|
||||
"source": f"ctx._source.censor_type={censor_type}; ctx._source.censoring_channel_hash=params[ctx._source.{key}]",
|
||||
"source": f"ctx._source.censor_type={censor_type}; ctx._source.censoring_channel_id=params[ctx._source.{key}]",
|
||||
"lang": "painless",
|
||||
"params": blockdict
|
||||
}
|
||||
@@ -208,7 +208,7 @@ class SearchIndex:
|
||||
|
||||
censored = [
|
||||
result if not isinstance(result, dict) or not censor.censor(result)
|
||||
else ResolveCensoredError(url, result['censoring_channel_hash'])
|
||||
else ResolveCensoredError(url, result['censoring_channel_id'])
|
||||
for url, result in zip(urls, results)
|
||||
]
|
||||
return results, censored, censor
|
||||
@@ -411,7 +411,7 @@ class SearchIndex:
|
||||
txo_rows = [row for row in txo_rows if isinstance(row, dict)]
|
||||
referenced_ids = set(filter(None, map(itemgetter('reposted_claim_id'), txo_rows)))
|
||||
referenced_ids |= set(filter(None, (row['channel_id'] for row in txo_rows)))
|
||||
referenced_ids |= set(map(parse_claim_id, filter(None, (row['censoring_channel_hash'] for row in txo_rows))))
|
||||
referenced_ids |= set(filter(None, (row['censoring_channel_id'] for row in txo_rows)))
|
||||
|
||||
referenced_txos = []
|
||||
if referenced_ids:
|
||||
@@ -432,20 +432,22 @@ def extract_doc(doc, index):
|
||||
doc['reposted_claim_id'] = None
|
||||
channel_hash = doc.pop('channel_hash')
|
||||
doc['channel_id'] = channel_hash[::-1].hex() if channel_hash else channel_hash
|
||||
channel_hash = doc.pop('censoring_channel_hash')
|
||||
doc['censoring_channel_hash'] = channel_hash[::-1].hex() if channel_hash else channel_hash
|
||||
doc['censoring_channel_id'] = doc.get('censoring_channel_id')
|
||||
txo_hash = doc.pop('txo_hash')
|
||||
doc['tx_id'] = txo_hash[:32][::-1].hex()
|
||||
doc['tx_nout'] = struct.unpack('<I', txo_hash[32:])[0]
|
||||
doc['repost_count'] = doc.pop('reposted')
|
||||
doc['is_controlling'] = bool(doc['is_controlling'])
|
||||
doc['signature'] = (doc.pop('signature') or b'').hex() or None
|
||||
doc['signature_digest'] = (doc.pop('signature_digest') or b'').hex() or None
|
||||
doc['public_key_bytes'] = (doc.pop('public_key_bytes') or b'').hex() or None
|
||||
doc['public_key_hash'] = (doc.pop('public_key_hash') or b'').hex() or None
|
||||
doc['signature_valid'] = bool(doc['signature_valid'])
|
||||
doc['public_key_id'] = (doc.pop('public_key_hash') or b'').hex() or None
|
||||
doc['is_signature_valid'] = bool(doc['signature_valid'])
|
||||
doc['claim_type'] = doc.get('claim_type', 0) or 0
|
||||
doc['stream_type'] = int(doc.get('stream_type', 0) or 0)
|
||||
doc['has_source'] = bool(doc['has_source'])
|
||||
doc['normalized_name'] = doc.pop('normalized')
|
||||
doc = {key: value for key, value in doc.items() if key in ALL_FIELDS}
|
||||
return {'doc': doc, '_id': doc['claim_id'], '_index': index, '_op_type': 'update', 'doc_as_upsert': True}
|
||||
|
||||
|
||||
@@ -463,6 +465,8 @@ def expand_query(**kwargs):
|
||||
for key, value in kwargs.items():
|
||||
key = key.replace('claim.', '')
|
||||
many = key.endswith('__in') or isinstance(value, list)
|
||||
if many and len(value) > 2048:
|
||||
raise TooManyClaimSearchParametersError(key, 2048)
|
||||
if many:
|
||||
key = key.replace('__in', '')
|
||||
value = list(filter(None, value))
|
||||
@@ -484,9 +488,8 @@ def expand_query(**kwargs):
|
||||
if not many and key in ('_id', 'claim_id') and len(value) < 20:
|
||||
partial_id = True
|
||||
if key == 'public_key_id':
|
||||
key = 'public_key_hash'
|
||||
value = Base58.decode(value)[1:21].hex()
|
||||
if key == 'signature_valid':
|
||||
if key in ('signature_valid', 'has_source'):
|
||||
continue # handled later
|
||||
if key in TEXT_FIELDS:
|
||||
key += '.keyword'
|
||||
@@ -536,12 +539,12 @@ def expand_query(**kwargs):
|
||||
if kwargs.get('has_channel_signature'):
|
||||
query['must'].append({"exists": {"field": "signature_digest"}})
|
||||
if 'signature_valid' in kwargs:
|
||||
query['must'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
|
||||
query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
|
||||
elif 'signature_valid' in kwargs:
|
||||
query.setdefault('should', [])
|
||||
query["minimum_should_match"] = 1
|
||||
query['should'].append({"bool": {"must_not": {"exists": {"field": "signature_digest"}}}})
|
||||
query['should'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
|
||||
query['should'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
|
||||
if 'has_source' in kwargs:
|
||||
query.setdefault('should', [])
|
||||
query["minimum_should_match"] = 1
|
||||
@@ -607,8 +610,9 @@ def expand_result(results):
|
||||
result['channel_hash'] = unhexlify(result['channel_id'])[::-1] if result['channel_id'] else None
|
||||
result['txo_hash'] = unhexlify(result['tx_id'])[::-1] + struct.pack('<I', result['tx_nout'])
|
||||
result['tx_hash'] = unhexlify(result['tx_id'])[::-1]
|
||||
if result['censoring_channel_hash']:
|
||||
result['censoring_channel_hash'] = unhexlify(result['censoring_channel_hash'])[::-1]
|
||||
result['reposted'] = result.pop('repost_count')
|
||||
result['signature_valid'] = result.pop('is_signature_valid')
|
||||
result['normalized'] = result.pop('normalized_name')
|
||||
expanded.append(result)
|
||||
if inner_hits:
|
||||
return expand_result(inner_hits)
|
||||
|
||||
@@ -22,21 +22,31 @@ async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
|
||||
db.row_factory = namedtuple_factory
|
||||
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
|
||||
for num, claim in enumerate(db.execute(f"""
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
""")):
|
||||
claim = dict(claim._asdict())
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_hash'] = None
|
||||
claim['censoring_channel_id'] = None
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
|
||||
if num % 10_000 == 0:
|
||||
|
||||
@@ -818,31 +818,42 @@ class SQLDB:
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
|
||||
"""
|
||||
for claim in self.execute(query):
|
||||
claim = claim._asdict()
|
||||
id_set = set(filter(None, (claim['claim_hash'], claim['channel_hash'], claim['reposted_claim_hash'])))
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_hash'] = None
|
||||
censoring_channel_hash = None
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
for reason_id in id_set:
|
||||
if reason_id in self.blocked_streams:
|
||||
claim['censor_type'] = 2
|
||||
claim['censoring_channel_hash'] = self.blocked_streams.get(reason_id)
|
||||
censoring_channel_hash = self.blocked_streams.get(reason_id)
|
||||
elif reason_id in self.blocked_channels:
|
||||
claim['censor_type'] = 2
|
||||
claim['censoring_channel_hash'] = self.blocked_channels.get(reason_id)
|
||||
censoring_channel_hash = self.blocked_channels.get(reason_id)
|
||||
elif reason_id in self.filtered_streams:
|
||||
claim['censor_type'] = 1
|
||||
claim['censoring_channel_hash'] = self.filtered_streams.get(reason_id)
|
||||
censoring_channel_hash = self.filtered_streams.get(reason_id)
|
||||
elif reason_id in self.filtered_channels:
|
||||
claim['censor_type'] = 1
|
||||
claim['censoring_channel_hash'] = self.filtered_channels.get(reason_id)
|
||||
censoring_channel_hash = self.filtered_channels.get(reason_id)
|
||||
claim['censoring_channel_id'] = censoring_channel_hash[::-1].hex() if censoring_channel_hash else None
|
||||
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
|
||||
|
||||
@@ -16,13 +16,13 @@ from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.util import pack_be_uint16, unpack_be_uint16_from
|
||||
from lbry.wallet.server.util import pack_be_uint32, unpack_be_uint32_from, unpack_be_uint16_from
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
|
||||
|
||||
class History:
|
||||
|
||||
DB_VERSIONS = [0]
|
||||
DB_VERSIONS = [0, 1]
|
||||
|
||||
def __init__(self):
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
@@ -32,9 +32,34 @@ class History:
|
||||
self.unflushed_count = 0
|
||||
self.db = None
|
||||
|
||||
@property
|
||||
def needs_migration(self):
|
||||
return self.db_version != max(self.DB_VERSIONS)
|
||||
|
||||
def migrate(self):
|
||||
# 0 -> 1: flush_count from 16 to 32 bits
|
||||
self.logger.warning("HISTORY MIGRATION IN PROGRESS. Please avoid shutting down before it finishes.")
|
||||
with self.db.write_batch() as batch:
|
||||
for key, value in self.db.iterator(prefix=b''):
|
||||
if len(key) != 13:
|
||||
continue
|
||||
flush_id, = unpack_be_uint16_from(key[-2:])
|
||||
new_key = key[:-2] + pack_be_uint32(flush_id)
|
||||
batch.put(new_key, value)
|
||||
self.logger.warning("history migration: new keys added, removing old ones.")
|
||||
for key, value in self.db.iterator(prefix=b''):
|
||||
if len(key) == 13:
|
||||
batch.delete(key)
|
||||
self.logger.warning("history migration: writing new state.")
|
||||
self.db_version = 1
|
||||
self.write_state(batch)
|
||||
self.logger.warning("history migration: done.")
|
||||
|
||||
def open_db(self, db_class, for_sync, utxo_flush_count, compacting):
|
||||
self.db = db_class('hist', for_sync)
|
||||
self.read_state()
|
||||
if self.needs_migration:
|
||||
self.migrate()
|
||||
self.clear_excess(utxo_flush_count)
|
||||
# An incomplete compaction needs to be cancelled otherwise
|
||||
# restarting it will corrupt the history
|
||||
@@ -81,7 +106,7 @@ class History:
|
||||
|
||||
keys = []
|
||||
for key, hist in self.db.iterator(prefix=b''):
|
||||
flush_id, = unpack_be_uint16_from(key[-2:])
|
||||
flush_id, = unpack_be_uint32_from(key[-4:])
|
||||
if flush_id > utxo_flush_count:
|
||||
keys.append(key)
|
||||
|
||||
@@ -126,7 +151,7 @@ class History:
|
||||
def flush(self):
|
||||
start_time = time.time()
|
||||
self.flush_count += 1
|
||||
flush_id = pack_be_uint16(self.flush_count)
|
||||
flush_id = pack_be_uint32(self.flush_count)
|
||||
unflushed = self.unflushed
|
||||
|
||||
with self.db.write_batch() as batch:
|
||||
@@ -250,7 +275,7 @@ class History:
|
||||
write_size = 0
|
||||
keys_to_delete.update(hist_map)
|
||||
for n, chunk in enumerate(util.chunks(full_hist, max_row_size)):
|
||||
key = hashX + pack_be_uint16(n)
|
||||
key = hashX + pack_be_uint32(n)
|
||||
if hist_map.get(key) == chunk:
|
||||
keys_to_delete.remove(key)
|
||||
else:
|
||||
@@ -301,8 +326,8 @@ class History:
|
||||
|
||||
# Loop over 2-byte prefixes
|
||||
cursor = self.comp_cursor
|
||||
while write_size < limit and cursor < 65536:
|
||||
prefix = pack_be_uint16(cursor)
|
||||
while write_size < limit and cursor < (1 << 32):
|
||||
prefix = pack_be_uint32(cursor)
|
||||
write_size += self._compact_prefix(prefix, write_items,
|
||||
keys_to_delete)
|
||||
cursor += 1
|
||||
|
||||
@@ -15,14 +15,13 @@ from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from elasticsearch import ConnectionTimeout
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.error import TooManyClaimSearchParametersError
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
@@ -1028,7 +1027,13 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
try:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
except TooManyClaimSearchParametersError as err:
|
||||
await asyncio.sleep(2)
|
||||
self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.",
|
||||
self.peer_address()[0], err.key, err.limit)
|
||||
return RPCError(1, str(err))
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
|
||||
@@ -2,7 +2,7 @@ import sys
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
|
||||
from lbry.wallet.server.db import SQLDB
|
||||
from lbry.wallet.server.db.writer import SQLDB
|
||||
from lbry.wallet.server.coin import LBC
|
||||
from lbry.wallet.server.daemon import Daemon
|
||||
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
"""A simple script that attempts to directly download a single blob.
|
||||
|
||||
To Do:
|
||||
------
|
||||
Currently `lbrynet blob get <hash>` does not work to download single blobs
|
||||
which are not already present in the system. The function locks up and
|
||||
never returns.
|
||||
It only works for blobs that are in the `blobfiles` directory already.
|
||||
|
||||
This bug is reported in lbryio/lbry-sdk, issue #2070.
|
||||
|
||||
Maybe this script can be investigated, and certain parts can be added to
|
||||
`lbry.extras.daemon.daemon.jsonrpc_blob_get`
|
||||
in order to solve the previous issue, and finally download single blobs
|
||||
from the network (peers or reflector servers).
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
@@ -47,7 +63,11 @@ async def main(blob_hash: str, url: str):
|
||||
print(f"deleted {blob_hash}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("usage: download_blob_from_peer.py <blob_hash> [host_url:port]")
|
||||
sys.exit(1)
|
||||
|
||||
url = 'reflector.lbry.com:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
from pprint import pprint
|
||||
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch._async.helpers import async_scan, async_bulk
|
||||
|
||||
DB = {}
|
||||
INDEX = 'claims'
|
||||
|
||||
|
||||
async def generate_support_amounts(client: AsyncElasticsearch):
|
||||
async for doc in async_scan(client):
|
||||
DB[doc['_id']] = doc['_source']['support_amount']
|
||||
if len(DB) > 10:
|
||||
break
|
||||
pprint(DB)
|
||||
|
||||
|
||||
def generate_support_to_trending():
|
||||
for claim_id, amount in DB.items():
|
||||
yield {'doc': {"trending_mixed": amount}, '_id': claim_id, '_index': INDEX, '_op_type': 'update'}
|
||||
|
||||
|
||||
async def write_trending(client: AsyncElasticsearch):
|
||||
await async_bulk(client, generate_support_to_trending())
|
||||
|
||||
|
||||
def get_client(host='localhost', port=9201):
|
||||
hosts = [{'host': host, 'port': port}]
|
||||
return AsyncElasticsearch(hosts, timeout=port)
|
||||
|
||||
|
||||
async def run():
|
||||
client = get_client()
|
||||
await generate_support_amounts(client)
|
||||
await write_trending(client)
|
||||
for claim_id, value in DB.items():
|
||||
if value > 0:
|
||||
break
|
||||
doc = await client.get(INDEX, claim_id)
|
||||
pprint(doc)
|
||||
pprint(DB[claim_id])
|
||||
await client.close()
|
||||
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(run())
|
||||
@@ -2,7 +2,7 @@ import time
|
||||
import asyncio
|
||||
import random
|
||||
from argparse import ArgumentParser
|
||||
from lbry.wallet.client.basenetwork import ClientSession
|
||||
from lbry.wallet.network import ClientSession
|
||||
|
||||
|
||||
class AgentSmith(ClientSession):
|
||||
|
||||
+6
-4
@@ -151,19 +151,21 @@ def release(args):
|
||||
for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry-sdk"):
|
||||
area_labels = list(get_labels(pr, 'area'))
|
||||
type_label = get_label(pr, 'type')
|
||||
pr_url = f'[#{pr.number}]({pr.html_url})'
|
||||
user_url = f'[{pr.user["login"]}]({pr.user["html_url"]})'
|
||||
if area_labels and type_label:
|
||||
for area_name in area_labels:
|
||||
for incompat in get_backwards_incompatible(pr.body or ""):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr_url})')
|
||||
for release_text in get_release_text(pr.body or ""):
|
||||
release_texts.append(release_text)
|
||||
if type_label == 'fixup':
|
||||
fixups.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
fixups.append(f' * {pr.title} ({pr_url}) by {user_url}')
|
||||
else:
|
||||
area = areas.setdefault(area_name, [])
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr_url}) by {user_url}')
|
||||
else:
|
||||
unlabeled.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
unlabeled.append(f' * {pr.title} ({pr_url}) by {user_url}')
|
||||
|
||||
area_names = list(areas.keys())
|
||||
area_names.sort()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import asyncio
|
||||
from lbry.wallet.client.basenetwork import ClientSession
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
import logging
|
||||
import json
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import asyncio
|
||||
from aiohttp import web
|
||||
|
||||
from lbry.blob_exchange.serialization import BlobRequest, BlobResponse
|
||||
from lbry.dht.constants import generate_id
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import make_kademlia_peer, PeerManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
NODE = Node(
|
||||
loop, PeerManager(loop), generate_id(), 60600, 60600, 3333, None,
|
||||
storage=SQLiteStorage(None, ":memory:", loop, loop.time)
|
||||
)
|
||||
|
||||
|
||||
async def check_p2p(ip, port):
|
||||
writer = None
|
||||
try:
|
||||
reader, writer = await asyncio.open_connection(ip, port)
|
||||
writer.write(BlobRequest.make_request_for_blob_hash('0'*96).serialize())
|
||||
return BlobResponse.deserialize(await reader.readuntil(b'}')).get_address_response().lbrycrd_address
|
||||
except OSError:
|
||||
return None
|
||||
finally:
|
||||
if writer:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
|
||||
async def check_dht(ip, port):
|
||||
peer = make_kademlia_peer(None, ip, udp_port=int(port))
|
||||
return await NODE.protocol.get_rpc_peer(peer).ping()
|
||||
|
||||
|
||||
async def endpoint_p2p(request):
|
||||
p2p_port = request.match_info.get('p2p_port', "3333")
|
||||
try:
|
||||
address = await asyncio.wait_for(check_p2p(request.remote, p2p_port), 3)
|
||||
except asyncio.TimeoutError:
|
||||
address = None
|
||||
return {"status": address is not None, "port": p2p_port, "payment_address": address}
|
||||
|
||||
|
||||
async def endpoint_dht(request):
|
||||
dht_port = request.match_info.get('dht_port', "3333")
|
||||
try:
|
||||
response = await check_dht(request.remote, dht_port)
|
||||
except asyncio.TimeoutError:
|
||||
response = None
|
||||
return {"status": response == b'pong', "port": dht_port}
|
||||
|
||||
|
||||
async def endpoint_default(request):
|
||||
return {"dht_status": await endpoint_dht(request), "p2p_status": await endpoint_p2p(request)}
|
||||
|
||||
|
||||
def as_json_response_wrapper(endpoint):
|
||||
async def json_endpoint(*args, **kwargs):
|
||||
return web.json_response(await endpoint(*args, **kwargs))
|
||||
return json_endpoint
|
||||
|
||||
|
||||
app = web.Application()
|
||||
app.add_routes([web.get('/', as_json_response_wrapper(endpoint_default)),
|
||||
web.get('/dht/{dht_port}', as_json_response_wrapper(endpoint_dht)),
|
||||
web.get('/p2p/{p2p_port}', as_json_response_wrapper(endpoint_p2p))])
|
||||
|
||||
if __name__ == '__main__':
|
||||
loop.create_task(NODE.start_listening("0.0.0.0"))
|
||||
web.run_app(app, port=60666)
|
||||
@@ -51,12 +51,17 @@ setup(
|
||||
'docopt==0.6.2',
|
||||
'hachoir',
|
||||
'multidict==4.6.1',
|
||||
'coincurve==11.0.0',
|
||||
'coincurve==15.0.0',
|
||||
'pbkdf2==1.3',
|
||||
'attrs==18.2.0',
|
||||
'pylru==1.1.0',
|
||||
'elasticsearch==7.10.1'
|
||||
'elasticsearch==7.10.1',
|
||||
] + PLYVEL,
|
||||
extras_require={
|
||||
'torrent': ['lbry-libtorrent'],
|
||||
'lint': ['mypy==0.701', 'pylint==2.4.4'],
|
||||
'test': ['coverage'],
|
||||
},
|
||||
classifiers=[
|
||||
'Framework :: AsyncIO',
|
||||
'Intended Audience :: Developers',
|
||||
|
||||
@@ -70,7 +70,9 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
async def assertFindsClaims(self, claims, **kwargs):
|
||||
kwargs.setdefault('order_by', ['height', '^name'])
|
||||
results = await self.claim_search(**kwargs)
|
||||
self.assertEqual(len(claims), len(results))
|
||||
self.assertEqual(
|
||||
len(claims), len(results),
|
||||
f"{[claim['outputs'][0]['name'] for claim in claims]} != {[result['name'] for result in results]}")
|
||||
for claim, result in zip(claims, results):
|
||||
self.assertEqual(
|
||||
(claim['txid'], self.get_claim_id(claim)),
|
||||
@@ -302,13 +304,15 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc')
|
||||
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc')
|
||||
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd')
|
||||
repost1 = await self.stream_repost(self.get_claim_id(claim1), 'repost1')
|
||||
repost5 = await self.stream_repost(self.get_claim_id(claim5), 'repost5')
|
||||
|
||||
await self.assertFindsClaims([claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
|
||||
await self.assertFindsClaims([claim4, claim3, claim2, claim1], fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
|
||||
await self.assertFindsClaims([repost5, repost1, claim5, claim4, claim3, claim2, claim1], fee_amount='>0')
|
||||
await self.assertFindsClaims([repost1, claim4, claim3, claim2, claim1], fee_currency='lbc')
|
||||
await self.assertFindsClaims([repost1, claim3, claim2, claim1], fee_amount='>0.1', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim5], fee_currency='usd')
|
||||
await self.assertFindsClaims([repost5, claim5], fee_currency='usd')
|
||||
|
||||
async def test_search_by_language(self):
|
||||
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
|
||||
@@ -474,7 +478,8 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
image_repost = await self.stream_repost(self.get_claim_id(image), 'image-repost')
|
||||
video_repost = await self.stream_repost(self.get_claim_id(video), 'video-repost')
|
||||
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
@@ -482,25 +487,25 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
await self.assertFindsClaims([video_repost, image_repost], claim_type='repost')
|
||||
await self.assertFindsClaims([collection], claim_type='collection')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
await self.assertFindsClaims([video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
await self.assertFindsClaims([video_repost, video], stream_types=['video'])
|
||||
await self.assertFindsClaims([image_repost, image], stream_types=['image'])
|
||||
await self.assertFindsClaims([video_repost, image_repost, image, video], stream_types=['video', 'image'])
|
||||
|
||||
# media_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
|
||||
await self.assertFindsClaims([video_repost, video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image_repost, image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([video_repost, image_repost, image, video], media_types=['video/mp4', 'image/png'])
|
||||
|
||||
# duration
|
||||
await self.assertFindsClaim(video, duration='>14')
|
||||
await self.assertFindsClaim(video, duration='<16')
|
||||
await self.assertFindsClaim(video, duration=15)
|
||||
await self.assertFindsClaims([video_repost, video], duration='>14')
|
||||
await self.assertFindsClaims([video_repost, video], duration='<16')
|
||||
await self.assertFindsClaims([video_repost, video], duration=15)
|
||||
await self.assertFindsClaims([], duration='>100')
|
||||
await self.assertFindsClaims([], duration='<14')
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.rpc import RPCError
|
||||
from lbry.wallet.server.db.elasticsearch.sync import run as run_sync, make_es_index
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
@@ -189,3 +190,19 @@ class TestHubDiscovery(CommandTestCase):
|
||||
('127.0.0.1', 9988): {}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestStress(CommandTestCase):
|
||||
async def test_flush_over_66_thousand(self):
|
||||
history = self.conductor.spv_node.server.db.history
|
||||
history.flush_count = 66_000
|
||||
history.flush()
|
||||
self.assertEqual(history.flush_count, 66_001)
|
||||
await self.generate(1)
|
||||
self.assertEqual(history.flush_count, 66_002)
|
||||
|
||||
async def test_thousands_claim_ids_on_search(self):
|
||||
await self.stream_create()
|
||||
with self.assertRaises(RPCError) as err:
|
||||
await self.claim_search(not_channel_ids=[("%040x" % i) for i in range(8196)])
|
||||
self.assertEqual(err.exception.message, 'not_channel_ids cant be set for more than 2048 items.')
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import time
|
||||
import unittest
|
||||
import typing
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
@@ -92,11 +93,12 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
|
||||
|
||||
class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
TIMEOUT = None # not supported as it advances time
|
||||
@unittest.SkipTest
|
||||
async def test_losing_connection(self):
|
||||
async def wait_for(check_ok, insist, timeout=20):
|
||||
start = loop.time()
|
||||
while loop.time() - start < timeout:
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
if check_ok():
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
@@ -159,5 +161,5 @@ class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
await advance(1000)
|
||||
await wait_for(
|
||||
lambda: len(node.protocol.routing_table.get_peers()) >= num_seeds,
|
||||
lambda: self.assertTrue(len(node.protocol.routing_table.get_peers()) >= num_seeds)
|
||||
lambda: self.assertGreaterEqual(len(node.protocol.routing_table.get_peers()), num_seeds)
|
||||
)
|
||||
@@ -9,6 +9,7 @@ from decimal import Decimal
|
||||
from lbry.file.file_manager import FileManager
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
from lbry.testcase import get_fake_exchange_rate_manager
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import InsufficientFundsError
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError, ResolveError, DownloadSDTimeoutError, DownloadDataTimeoutError
|
||||
from lbry.wallet import WalletManager, Wallet, Ledger, Transaction, Input, Output, Database
|
||||
@@ -139,7 +140,11 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.sd_hash = descriptor.sd_hash
|
||||
self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, self.client_wallet_dir,
|
||||
balance, fee)
|
||||
analytics_manager = AnalyticsManager(self.client_config)
|
||||
analytics_manager = AnalyticsManager(
|
||||
self.client_config,
|
||||
binascii.hexlify(generate_id()).decode(),
|
||||
binascii.hexlify(generate_id()).decode()
|
||||
)
|
||||
self.stream_manager = StreamManager(
|
||||
self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
|
||||
self.client_storage, get_mock_node(self.server_from_client),
|
||||
|
||||
@@ -414,7 +414,7 @@ def resolve(urls) -> Tuple[List, List]:
|
||||
txo_rows = [resolve_url(raw_url) for raw_url in urls]
|
||||
extra_txo_rows = _get_referenced_rows(
|
||||
[txo for txo in txo_rows if isinstance(txo, dict)],
|
||||
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
|
||||
[txo.censor_id for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
|
||||
)
|
||||
return txo_rows, extra_txo_rows
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import unittest
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
|
||||
from lbry.wallet.server.history import History
|
||||
from lbry.wallet.server.storage import LevelDB
|
||||
|
||||
|
||||
# dumped from a real history database. Aside from the state, all records are <hashX><flush_count>: <value>
|
||||
STATE_RECORD = (b'state\x00\x00', b"{'flush_count': 21497, 'comp_flush_count': -1, 'comp_cursor': -1, 'db_version': 0}")
|
||||
UNMIGRATED_RECORDS = {
|
||||
'00538b2cbe4a5f1be2dc320241': 'f5ed500142ee5001',
|
||||
'00538b48def1904014880501f2': 'b9a52a01baa52a01',
|
||||
'00538cdcf989b74de32c5100ca': 'c973870078748700',
|
||||
'00538d42d5df44603474284ae1': 'f5d9d802',
|
||||
'00538d42d5df44603474284ae2': '75dad802',
|
||||
'00538ebc879dac6ddbee9e0029': '3ca42f0042a42f00',
|
||||
'00538ed1d391327208748200bc': '804e7d00af4e7d00',
|
||||
'00538f3de41d9e33affa0300c2': '7de8810086e88100',
|
||||
'00539007f87792d98422c505a5': '8c5a7202445b7202',
|
||||
'0053902cf52ee9682d633b0575': 'eb0f64026c106402',
|
||||
'005390e05674571551632205a2': 'a13d7102e13d7102',
|
||||
'0053914ef25a9ceed927330584': '78096902960b6902',
|
||||
'005391768113f69548f37a01b1': 'a5b90b0114ba0b01',
|
||||
'005391a289812669e5b44c02c2': '33da8a016cdc8a01',
|
||||
}
|
||||
|
||||
|
||||
class TestHistoryDBMigration(unittest.TestCase):
|
||||
def test_migrate_flush_count_from_16_to_32_bits(self):
|
||||
self.history = History()
|
||||
tmpdir = mkdtemp()
|
||||
self.addCleanup(lambda: rmtree(tmpdir))
|
||||
LevelDB.import_module()
|
||||
db = LevelDB(tmpdir, 'hist', True)
|
||||
with db.write_batch() as batch:
|
||||
for key, value in UNMIGRATED_RECORDS.items():
|
||||
batch.put(bytes.fromhex(key), bytes.fromhex(value))
|
||||
batch.put(*STATE_RECORD)
|
||||
self.history.db = db
|
||||
self.history.read_state()
|
||||
self.assertEqual(21497, self.history.flush_count)
|
||||
self.assertEqual(0, self.history.db_version)
|
||||
self.assertTrue(self.history.needs_migration)
|
||||
self.history.migrate()
|
||||
self.assertFalse(self.history.needs_migration)
|
||||
self.assertEqual(1, self.history.db_version)
|
||||
for idx, (key, value) in enumerate(sorted(db.iterator())):
|
||||
if key == b'state\x00\x00':
|
||||
continue
|
||||
key, counter = key[:-4], key[-4:]
|
||||
expected_value = UNMIGRATED_RECORDS[key.hex() + counter.hex()[-4:]]
|
||||
self.assertEqual(value.hex(), expected_value)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from lbry.testcase import AsyncioTestCase
|
||||
from lbry.wallet import Wallet, Ledger, Database, Headers, Account, SingleKey, HierarchicalDeterministic
|
||||
@@ -37,6 +38,18 @@ class TestAccount(AsyncioTestCase):
|
||||
addresses = await account.change.get_addresses()
|
||||
self.assertEqual(len(addresses), 6)
|
||||
|
||||
async def test_unused_address_on_account_creation_does_not_cause_a_race(self):
|
||||
account = Account.generate(self.ledger, Wallet(), 'lbryum')
|
||||
await account.ledger.db.db.executescript("update pubkey_address set used_times=10")
|
||||
await account.receiving.address_generator_lock.acquire()
|
||||
delayed1 = asyncio.ensure_future(account.receiving.ensure_address_gap())
|
||||
delayed = asyncio.ensure_future(account.receiving.get_or_create_usable_address())
|
||||
await asyncio.sleep(0)
|
||||
# wallet being created and queried at the same time
|
||||
account.receiving.address_generator_lock.release()
|
||||
await delayed1
|
||||
await delayed
|
||||
|
||||
async def test_generate_keys_over_batch_threshold_saves_it_properly(self):
|
||||
account = Account.generate(self.ledger, Wallet(), 'lbryum')
|
||||
async with account.receiving.address_generator_lock:
|
||||
|
||||
@@ -470,7 +470,7 @@ class TestUpgrade(AsyncioTestCase):
|
||||
|
||||
|
||||
class TestSQLiteRace(AsyncioTestCase):
|
||||
max_misuse_attempts = 40000
|
||||
max_misuse_attempts = 80000
|
||||
|
||||
def setup_db(self):
|
||||
self.db = sqlite3.connect(":memory:", isolation_level=None)
|
||||
|
||||
@@ -1,18 +1,14 @@
|
||||
[testenv]
|
||||
deps =
|
||||
coverage
|
||||
|
||||
extras = test
|
||||
extras =
|
||||
test
|
||||
torrent
|
||||
changedir = {toxinidir}/tests
|
||||
setenv =
|
||||
HOME=/tmp
|
||||
ELASTIC_HOST={env:ELASTIC_HOST:localhost}
|
||||
commands =
|
||||
pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
--global-option=build --global-option=--enable --global-option=fts5
|
||||
pip install lbry-libtorrent
|
||||
orchstr8 download
|
||||
blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs}
|
||||
datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs}
|
||||
|
||||
Reference in new issue
Block a user