Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e01fbc6f39 |
No files matched your search
@@ -1,13 +1,9 @@
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
/lbry-venv
|
||||
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
@@ -33,7 +33,9 @@ unsafe-load-any-extension=no
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
# extension-pkg-whitelist=
|
||||
extension-pkg-whitelist=
|
||||
miniupnpc,
|
||||
unqlite
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
|
||||
+15
-14
@@ -9,7 +9,9 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid pylint
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
@@ -35,17 +37,6 @@ jobs:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
@@ -80,7 +71,7 @@ jobs:
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python3 scripts/set_build.py
|
||||
- python scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
@@ -115,6 +106,16 @@ jobs:
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
@@ -123,7 +124,7 @@ jobs:
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
- if: tag IS blank AND branch = master
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.tech/contribute
|
||||
https://lbry.io/faq/contributing
|
||||
+12
-11
@@ -13,6 +13,15 @@ Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
@@ -20,17 +29,15 @@ macOS users will need to install [xcode command line tools](https://developer.xa
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python protobuf
|
||||
brew install python3 protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
@@ -45,14 +52,8 @@ To install:
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
Creating venv:
|
||||
python -m venv lbry-venv
|
||||
|
||||
Activating lbry-venv on Linux/Mac:
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
Activating lbry-venv on Windows:
|
||||
lbry-venv\Scripts\activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
Copyright (c) 2015-2018 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
|
||||
|
||||
## License
|
||||
|
||||
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
build:
|
||||
python setup.py apk
|
||||
adb install -r lbry-0.1-service-debug.apk
|
||||
|
||||
.PHONY: build
|
||||
@@ -0,0 +1,49 @@
|
||||
import inspect
|
||||
from os.path import dirname, join
|
||||
|
||||
import sh
|
||||
from pythonforandroid.util import ensure_dir
|
||||
from pythonforandroid.toolchain import (
|
||||
Bootstrap, current_directory, info, info_main, shprint
|
||||
)
|
||||
|
||||
|
||||
class LBRYServiceBootstrap(Bootstrap):
|
||||
name = 'lbry-service'
|
||||
recipe_depends = ['genericndkbuild', 'python3']
|
||||
bootstrap_dir = dirname(__file__)
|
||||
|
||||
def get_common_dir(self):
|
||||
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
|
||||
|
||||
def run_distribute(self):
|
||||
info_main('# Creating Android project from build and {} bootstrap'.format(
|
||||
self.name))
|
||||
|
||||
info('This currently just copies the build stuff straight from the build dir.')
|
||||
shprint(sh.rm, '-rf', self.dist_dir)
|
||||
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
|
||||
with current_directory(self.dist_dir):
|
||||
with open('local.properties', 'w') as fileh:
|
||||
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
|
||||
|
||||
arch = self.ctx.archs[0]
|
||||
if len(self.ctx.archs) > 1:
|
||||
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
|
||||
info('Bootstrap running with arch {}'.format(arch))
|
||||
|
||||
with current_directory(self.dist_dir):
|
||||
info('Copying python distribution')
|
||||
|
||||
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
|
||||
self.distribute_aars(arch)
|
||||
self.distribute_javaclasses(self.ctx.javaclass_dir)
|
||||
|
||||
python_bundle_dir = join('_python_bundle', '_python_bundle')
|
||||
ensure_dir(python_bundle_dir)
|
||||
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
|
||||
join(self.dist_dir, python_bundle_dir), arch)
|
||||
|
||||
self.strip_libraries(arch)
|
||||
self.fry_eggs(site_packages_dir)
|
||||
super().run_distribute()
|
||||
@@ -0,0 +1,89 @@
|
||||
# prevent user to include invalid extensions
|
||||
*.apk
|
||||
*.pxd
|
||||
|
||||
# eggs
|
||||
*.egg-info
|
||||
|
||||
# unit test
|
||||
unittest/*
|
||||
|
||||
# python config
|
||||
config/makesetup
|
||||
|
||||
# unused kivy files (platform specific)
|
||||
kivy/input/providers/wm_*
|
||||
kivy/input/providers/mactouch*
|
||||
kivy/input/providers/probesysfs*
|
||||
kivy/input/providers/mtdev*
|
||||
kivy/input/providers/hidinput*
|
||||
kivy/core/camera/camera_videocapture*
|
||||
kivy/core/spelling/*osx*
|
||||
kivy/core/video/video_pyglet*
|
||||
kivy/tools
|
||||
kivy/tests/*
|
||||
kivy/*/*.h
|
||||
kivy/*/*.pxi
|
||||
|
||||
# unused encodings
|
||||
lib-dynload/*codec*
|
||||
encodings/cp*.pyo
|
||||
encodings/tis*
|
||||
encodings/shift*
|
||||
encodings/bz2*
|
||||
encodings/iso*
|
||||
encodings/undefined*
|
||||
encodings/johab*
|
||||
encodings/p*
|
||||
encodings/m*
|
||||
encodings/euc*
|
||||
encodings/k*
|
||||
encodings/unicode_internal*
|
||||
encodings/quo*
|
||||
encodings/gb*
|
||||
encodings/big5*
|
||||
encodings/hp*
|
||||
encodings/hz*
|
||||
|
||||
# unused python modules
|
||||
bsddb/*
|
||||
wsgiref/*
|
||||
hotshot/*
|
||||
pydoc_data/*
|
||||
tty.pyo
|
||||
anydbm.pyo
|
||||
nturl2path.pyo
|
||||
LICENCE.txt
|
||||
macurl2path.pyo
|
||||
dummy_threading.pyo
|
||||
audiodev.pyo
|
||||
antigravity.pyo
|
||||
dumbdbm.pyo
|
||||
sndhdr.pyo
|
||||
__phello__.foo.pyo
|
||||
sunaudio.pyo
|
||||
os2emxpath.pyo
|
||||
multiprocessing/dummy*
|
||||
|
||||
# unused binaries python modules
|
||||
lib-dynload/termios.so
|
||||
lib-dynload/_lsprof.so
|
||||
lib-dynload/*audioop.so
|
||||
lib-dynload/_hotshot.so
|
||||
lib-dynload/_heapq.so
|
||||
lib-dynload/_json.so
|
||||
lib-dynload/grp.so
|
||||
lib-dynload/resource.so
|
||||
lib-dynload/pyexpat.so
|
||||
lib-dynload/_ctypes_test.so
|
||||
lib-dynload/_testcapi.so
|
||||
|
||||
# odd files
|
||||
plat-linux3/regen
|
||||
|
||||
#>sqlite3
|
||||
# conditionnal include depending if some recipes are included or not.
|
||||
sqlite3/*
|
||||
lib-dynload/_sqlite3.so
|
||||
#<sqlite3
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
include $(call all-subdir-makefiles)
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
# Uncomment this if you're using STL in your project
|
||||
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
|
||||
# APP_STL := stlport_static
|
||||
|
||||
# APP_ABI := armeabi armeabi-v7a x86
|
||||
APP_ABI := $(ARCH)
|
||||
@@ -0,0 +1,22 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
# Add your application source files here...
|
||||
LOCAL_SRC_FILES := start.c pyjniusjni.c
|
||||
|
||||
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
|
||||
|
||||
LOCAL_SHARED_LIBRARIES := python_shared
|
||||
|
||||
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
|
||||
|
||||
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
|
||||
ifdef CRYSTAX_PYTHON_VERSION
|
||||
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
|
||||
endif
|
||||
@@ -0,0 +1,10 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
LOCAL_SRC_FILES := YourSourceHere.c
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
#define BOOTSTRAP_NAME_SERVICEONLY
|
||||
#define BOOTSTRAP_USES_NO_SDL_HEADERS
|
||||
|
||||
const char bootstrap_name[] = "service_only";
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
#include <pthread.h>
|
||||
#include <jni.h>
|
||||
|
||||
#define LOGI(...) do {} while (0)
|
||||
#define LOGE(...) do {} while (0)
|
||||
|
||||
#include "android/log.h"
|
||||
|
||||
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
|
||||
|
||||
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
|
||||
/* #define LOGP(x) LOG("python", (x)) */
|
||||
#define LOG_TAG "Python_android"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
|
||||
|
||||
|
||||
/* Function headers */
|
||||
JNIEnv* Android_JNI_GetEnv(void);
|
||||
static void Android_JNI_ThreadDestroyed(void*);
|
||||
|
||||
static pthread_key_t mThreadKey;
|
||||
static JavaVM* mJavaVM;
|
||||
|
||||
int Android_JNI_SetupThread(void)
|
||||
{
|
||||
Android_JNI_GetEnv();
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Library init */
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
|
||||
{
|
||||
JNIEnv *env;
|
||||
mJavaVM = vm;
|
||||
LOGI("JNI_OnLoad called");
|
||||
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
|
||||
LOGE("Failed to get the environment using GetEnv()");
|
||||
return -1;
|
||||
}
|
||||
/*
|
||||
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
|
||||
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
|
||||
*/
|
||||
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
|
||||
|
||||
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
|
||||
}
|
||||
Android_JNI_SetupThread();
|
||||
|
||||
return JNI_VERSION_1_4;
|
||||
}
|
||||
|
||||
JNIEnv* Android_JNI_GetEnv(void)
|
||||
{
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* All threads are Linux threads, scheduled by the kernel.
|
||||
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
|
||||
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
|
||||
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
|
||||
* and cannot make JNI calls.
|
||||
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
|
||||
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
|
||||
* is a no-op.
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
*/
|
||||
|
||||
JNIEnv *env;
|
||||
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
|
||||
if(status < 0) {
|
||||
LOGE("failed to attach current thread");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
|
||||
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
|
||||
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
|
||||
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
|
||||
* Note: The destructor is not called unless the stored value is != NULL
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
* (except for some lost CPU cycles)
|
||||
*/
|
||||
pthread_setspecific(mThreadKey, (void*) env);
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
static void Android_JNI_ThreadDestroyed(void* value)
|
||||
{
|
||||
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
|
||||
JNIEnv *env = (JNIEnv*) value;
|
||||
if (env != NULL) {
|
||||
(*mJavaVM)->DetachCurrentThread(mJavaVM);
|
||||
pthread_setspecific(mThreadKey, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
void *WebView_AndroidGetJNIEnv()
|
||||
{
|
||||
return Android_JNI_GetEnv();
|
||||
}
|
||||
File renamed without changes.
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class Octal {
|
||||
|
||||
/**
|
||||
* Parse an octal string from a header buffer. This is used for the file
|
||||
* permission mode value.
|
||||
*
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal string.
|
||||
*/
|
||||
public static long parseOctal(byte[] header, int offset, int length) {
|
||||
long result = 0;
|
||||
boolean stillPadding = true;
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
|
||||
if (header[i] == (byte) ' ' || header[i] == '0') {
|
||||
if (stillPadding)
|
||||
continue;
|
||||
|
||||
if (header[i] == (byte) ' ')
|
||||
break;
|
||||
}
|
||||
|
||||
stillPadding = false;
|
||||
|
||||
result = ( result << 3 ) + ( header[i] - '0' );
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The integer value of the octal bytes.
|
||||
*/
|
||||
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
int idx = length - 1;
|
||||
|
||||
buf[offset + idx] = 0;
|
||||
--idx;
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
--idx;
|
||||
|
||||
if (value == 0) {
|
||||
buf[offset + idx] = (byte) '0';
|
||||
--idx;
|
||||
} else {
|
||||
for (long val = value; idx >= 0 && val > 0; --idx) {
|
||||
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
|
||||
val = val >> 3;
|
||||
}
|
||||
}
|
||||
|
||||
for (; idx >= 0; --idx) {
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the checksum octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The integer value of the entry's checksum.
|
||||
*/
|
||||
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
getOctalBytes( value, buf, offset, length );
|
||||
buf[offset + length - 1] = (byte) ' ';
|
||||
buf[offset + length - 2] = 0;
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal long integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal bytes.
|
||||
*/
|
||||
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
byte[] temp = new byte[length + 1];
|
||||
getOctalBytes( value, temp, 0, length + 1 );
|
||||
System.arraycopy( temp, 0, buf, offset, length );
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarConstants {
|
||||
public static final int EOF_BLOCK = 1024;
|
||||
public static final int DATA_BLOCK = 512;
|
||||
public static final int HEADER_BLOCK = 512;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarEntry {
|
||||
protected File file;
|
||||
protected TarHeader header;
|
||||
|
||||
private TarEntry() {
|
||||
this.file = null;
|
||||
header = new TarHeader();
|
||||
}
|
||||
|
||||
public TarEntry(File file, String entryName) {
|
||||
this();
|
||||
this.file = file;
|
||||
this.extractTarHeader(entryName);
|
||||
}
|
||||
|
||||
public TarEntry(byte[] headerBuf) {
|
||||
this();
|
||||
this.parseTarHeader(headerBuf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor to create an entry from an existing TarHeader object.
|
||||
*
|
||||
* This method is useful to add new entries programmatically (e.g. for
|
||||
* adding files or directories that do not exist in the file system).
|
||||
*
|
||||
* @param header
|
||||
*
|
||||
*/
|
||||
public TarEntry(TarHeader header) {
|
||||
this.file = null;
|
||||
this.header = header;
|
||||
}
|
||||
|
||||
public boolean equals(TarEntry it) {
|
||||
return header.name.toString().equals(it.header.name.toString());
|
||||
}
|
||||
|
||||
public boolean isDescendent(TarEntry desc) {
|
||||
return desc.header.name.toString().startsWith(header.name.toString());
|
||||
}
|
||||
|
||||
public TarHeader getHeader() {
|
||||
return header;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
String name = header.name.toString();
|
||||
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
|
||||
name = header.namePrefix.toString() + "/" + name;
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
public int getUserId() {
|
||||
return header.userId;
|
||||
}
|
||||
|
||||
public void setUserId(int userId) {
|
||||
header.userId = userId;
|
||||
}
|
||||
|
||||
public int getGroupId() {
|
||||
return header.groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(int groupId) {
|
||||
header.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return header.userName.toString();
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
header.userName = new StringBuffer(userName);
|
||||
}
|
||||
|
||||
public String getGroupName() {
|
||||
return header.groupName.toString();
|
||||
}
|
||||
|
||||
public void setGroupName(String groupName) {
|
||||
header.groupName = new StringBuffer(groupName);
|
||||
}
|
||||
|
||||
public void setIds(int userId, int groupId) {
|
||||
this.setUserId(userId);
|
||||
this.setGroupId(groupId);
|
||||
}
|
||||
|
||||
public void setModTime(long time) {
|
||||
header.modTime = time / 1000;
|
||||
}
|
||||
|
||||
public void setModTime(Date time) {
|
||||
header.modTime = time.getTime() / 1000;
|
||||
}
|
||||
|
||||
public Date getModTime() {
|
||||
return new Date(header.modTime * 1000);
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return this.file;
|
||||
}
|
||||
|
||||
public long getSize() {
|
||||
return header.size;
|
||||
}
|
||||
|
||||
public void setSize(long size) {
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the org.kamrazafar.jtar entry is a directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean isDirectory() {
|
||||
if (this.file != null)
|
||||
return this.file.isDirectory();
|
||||
|
||||
if (header != null) {
|
||||
if (header.linkFlag == TarHeader.LF_DIR)
|
||||
return true;
|
||||
|
||||
if (header.name.toString().endsWith("/"))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract header from File
|
||||
*
|
||||
* @param entryName
|
||||
*/
|
||||
public void extractTarHeader(String entryName) {
|
||||
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum
|
||||
*
|
||||
* @param buf
|
||||
* @return
|
||||
*/
|
||||
public long computeCheckSum(byte[] buf) {
|
||||
long sum = 0;
|
||||
|
||||
for (int i = 0; i < buf.length; ++i) {
|
||||
sum += 255 & buf[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the header to the byte buffer
|
||||
*
|
||||
* @param outbuf
|
||||
*/
|
||||
public void writeEntryHeader(byte[] outbuf) {
|
||||
int offset = 0;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
|
||||
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
|
||||
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
|
||||
|
||||
long size = header.size;
|
||||
|
||||
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
|
||||
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
|
||||
|
||||
int csOffset = offset;
|
||||
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
|
||||
outbuf[offset++] = (byte) ' ';
|
||||
|
||||
outbuf[offset++] = header.linkFlag;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
|
||||
for (; offset < outbuf.length;)
|
||||
outbuf[offset++] = 0;
|
||||
|
||||
long checkSum = this.computeCheckSum(outbuf);
|
||||
|
||||
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the tar header to the byte buffer
|
||||
*
|
||||
* @param header
|
||||
* @param bh
|
||||
*/
|
||||
public void parseTarHeader(byte[] bh) {
|
||||
int offset = 0;
|
||||
|
||||
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
|
||||
offset += TarHeader.MODELEN;
|
||||
|
||||
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
|
||||
offset += TarHeader.UIDLEN;
|
||||
|
||||
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
|
||||
offset += TarHeader.GIDLEN;
|
||||
|
||||
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
|
||||
offset += TarHeader.SIZELEN;
|
||||
|
||||
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
|
||||
offset += TarHeader.MODTIMELEN;
|
||||
|
||||
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
|
||||
offset += TarHeader.CHKSUMLEN;
|
||||
|
||||
header.linkFlag = bh[offset++];
|
||||
|
||||
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset += TarHeader.USTAR_MAGICLEN;
|
||||
|
||||
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset += TarHeader.USTAR_USER_NAMELEN;
|
||||
|
||||
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset += TarHeader.USTAR_GROUP_NAMELEN;
|
||||
|
||||
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 0 100 File name
|
||||
* 100 8 File mode
|
||||
* 108 8 Owner's numeric user ID
|
||||
* 116 8 Group's numeric user ID
|
||||
* 124 12 File size in bytes
|
||||
* 136 12 Last modification time in numeric Unix time format
|
||||
* 148 8 Checksum for header block
|
||||
* 156 1 Link indicator (file type)
|
||||
* 157 100 Name of linked file
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* File Types
|
||||
*
|
||||
* <pre>
|
||||
* Value Meaning
|
||||
* '0' Normal file
|
||||
* (ASCII NUL) Normal file (now obsolete)
|
||||
* '1' Hard link
|
||||
* '2' Symbolic link
|
||||
* '3' Character special
|
||||
* '4' Block special
|
||||
* '5' Directory
|
||||
* '6' FIFO
|
||||
* '7' Contigous
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
*
|
||||
* Ustar header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 257 6 UStar indicator "ustar"
|
||||
* 263 2 UStar version "00"
|
||||
* 265 32 Owner user name
|
||||
* 297 32 Owner group name
|
||||
* 329 8 Device major number
|
||||
* 337 8 Device minor number
|
||||
* 345 155 Filename prefix
|
||||
* </pre>
|
||||
*/
|
||||
|
||||
public class TarHeader {
|
||||
|
||||
/*
|
||||
* Header
|
||||
*/
|
||||
public static final int NAMELEN = 100;
|
||||
public static final int MODELEN = 8;
|
||||
public static final int UIDLEN = 8;
|
||||
public static final int GIDLEN = 8;
|
||||
public static final int SIZELEN = 12;
|
||||
public static final int MODTIMELEN = 12;
|
||||
public static final int CHKSUMLEN = 8;
|
||||
public static final byte LF_OLDNORM = 0;
|
||||
|
||||
/*
|
||||
* File Types
|
||||
*/
|
||||
public static final byte LF_NORMAL = (byte) '0';
|
||||
public static final byte LF_LINK = (byte) '1';
|
||||
public static final byte LF_SYMLINK = (byte) '2';
|
||||
public static final byte LF_CHR = (byte) '3';
|
||||
public static final byte LF_BLK = (byte) '4';
|
||||
public static final byte LF_DIR = (byte) '5';
|
||||
public static final byte LF_FIFO = (byte) '6';
|
||||
public static final byte LF_CONTIG = (byte) '7';
|
||||
|
||||
/*
|
||||
* Ustar header
|
||||
*/
|
||||
|
||||
public static final String USTAR_MAGIC = "ustar"; // POSIX
|
||||
|
||||
public static final int USTAR_MAGICLEN = 8;
|
||||
public static final int USTAR_USER_NAMELEN = 32;
|
||||
public static final int USTAR_GROUP_NAMELEN = 32;
|
||||
public static final int USTAR_DEVLEN = 8;
|
||||
public static final int USTAR_FILENAME_PREFIX = 155;
|
||||
|
||||
// Header values
|
||||
public StringBuffer name;
|
||||
public int mode;
|
||||
public int userId;
|
||||
public int groupId;
|
||||
public long size;
|
||||
public long modTime;
|
||||
public int checkSum;
|
||||
public byte linkFlag;
|
||||
public StringBuffer linkName;
|
||||
public StringBuffer magic; // ustar indicator and version
|
||||
public StringBuffer userName;
|
||||
public StringBuffer groupName;
|
||||
public int devMajor;
|
||||
public int devMinor;
|
||||
public StringBuffer namePrefix;
|
||||
|
||||
public TarHeader() {
|
||||
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
|
||||
|
||||
this.name = new StringBuffer();
|
||||
this.linkName = new StringBuffer();
|
||||
|
||||
String user = System.getProperty("user.name", "");
|
||||
|
||||
if (user.length() > 31)
|
||||
user = user.substring(0, 31);
|
||||
|
||||
this.userId = 0;
|
||||
this.groupId = 0;
|
||||
this.userName = new StringBuffer(user);
|
||||
this.groupName = new StringBuffer("");
|
||||
this.namePrefix = new StringBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an entry name from a header buffer.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The header's entry name.
|
||||
*/
|
||||
public static StringBuffer parseName(byte[] header, int offset, int length) {
|
||||
StringBuffer result = new StringBuffer(length);
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
result.append((char) header[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the number of bytes in an entry name.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The number of bytes in a header's entry name.
|
||||
*/
|
||||
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < length && i < name.length(); ++i) {
|
||||
buf[offset + i] = (byte) name.charAt(i);
|
||||
}
|
||||
|
||||
for (; i < length; ++i) {
|
||||
buf[offset + i] = 0;
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new header for a file/directory entry.
|
||||
*
|
||||
*
|
||||
* @param name
|
||||
* File name
|
||||
* @param size
|
||||
* File size in bytes
|
||||
* @param modTime
|
||||
* Last modification time in numeric Unix time format
|
||||
* @param dir
|
||||
* Is directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
|
||||
String name = entryName;
|
||||
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
|
||||
|
||||
TarHeader header = new TarHeader();
|
||||
header.linkName = new StringBuffer("");
|
||||
|
||||
if (name.length() > 100) {
|
||||
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
|
||||
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
|
||||
} else {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
if (dir) {
|
||||
header.mode = 040755;
|
||||
header.linkFlag = TarHeader.LF_DIR;
|
||||
if (header.name.charAt(header.name.length() - 1) != '/') {
|
||||
header.name.append("/");
|
||||
}
|
||||
header.size = 0;
|
||||
} else {
|
||||
header.mode = 0100644;
|
||||
header.linkFlag = TarHeader.LF_NORMAL;
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
header.modTime = modTime;
|
||||
header.checkSum = 0;
|
||||
header.devMajor = 0;
|
||||
header.devMinor = 0;
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
+249
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarInputStream extends FilterInputStream {
|
||||
|
||||
private static final int SKIP_BUFFER_SIZE = 2048;
|
||||
private TarEntry currentEntry;
|
||||
private long currentFileSize;
|
||||
private long bytesRead;
|
||||
private boolean defaultSkip = false;
|
||||
|
||||
public TarInputStream(InputStream in) {
|
||||
super(in);
|
||||
currentFileSize = 0;
|
||||
bytesRead = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void mark(int readlimit) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void reset() throws IOException {
|
||||
throw new IOException("mark/reset not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a byte
|
||||
*
|
||||
* @see java.io.FilterInputStream#read()
|
||||
*/
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
byte[] buf = new byte[1];
|
||||
|
||||
int res = this.read(buf, 0, 1);
|
||||
|
||||
if (res != -1) {
|
||||
return 0xFF & buf[0];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being read exceed the entry size and adjusts the byte
|
||||
* array length. Updates the byte counters
|
||||
*
|
||||
*
|
||||
* @see java.io.FilterInputStream#read(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentFileSize == currentEntry.getSize()) {
|
||||
return -1;
|
||||
} else if ((currentEntry.getSize() - currentFileSize) < len) {
|
||||
len = (int) (currentEntry.getSize() - currentFileSize);
|
||||
}
|
||||
}
|
||||
|
||||
int br = super.read(b, off, len);
|
||||
|
||||
if (br != -1) {
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += br;
|
||||
}
|
||||
|
||||
bytesRead += br;
|
||||
}
|
||||
|
||||
return br;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next entry in the tar file
|
||||
*
|
||||
* @return TarEntry
|
||||
* @throws IOException
|
||||
*/
|
||||
public TarEntry getNextEntry() throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
|
||||
int tr = 0;
|
||||
|
||||
// Read full header
|
||||
while (tr < TarConstants.HEADER_BLOCK) {
|
||||
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
|
||||
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
System.arraycopy(theader, 0, header, tr, res);
|
||||
tr += res;
|
||||
}
|
||||
|
||||
// Check if record is null
|
||||
boolean eof = true;
|
||||
for (byte b : header) {
|
||||
if (b != 0) {
|
||||
eof = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!eof) {
|
||||
currentEntry = new TarEntry(header);
|
||||
}
|
||||
|
||||
return currentEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current offset (in bytes) from the beginning of the stream.
|
||||
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
|
||||
*/
|
||||
public long getCurrentOffset() {
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
// Not fully read, skip rest of the bytes
|
||||
long bs = 0;
|
||||
while (bs < currentEntry.getSize() - currentFileSize) {
|
||||
long res = skip(currentEntry.getSize() - currentFileSize - bs);
|
||||
|
||||
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
|
||||
// I suspect file corruption
|
||||
throw new IOException("Possible tar file corruption");
|
||||
}
|
||||
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0L;
|
||||
skipPad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips the pad at the end of each tar entry file content
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void skipPad() throws IOException {
|
||||
if (bytesRead > 0) {
|
||||
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
|
||||
|
||||
if (extra > 0) {
|
||||
long bs = 0;
|
||||
while (bs < TarConstants.DATA_BLOCK - extra) {
|
||||
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips 'n' bytes on the InputStream<br>
|
||||
* Overrides default implementation of skip
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
if (defaultSkip) {
|
||||
// use skip method of parent stream
|
||||
// may not work if skip not implemented by parent
|
||||
long bs = super.skip(n);
|
||||
bytesRead += bs;
|
||||
|
||||
return bs;
|
||||
}
|
||||
|
||||
if (n <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
long left = n;
|
||||
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
|
||||
|
||||
while (left > 0) {
|
||||
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
left -= res;
|
||||
}
|
||||
|
||||
return n - left;
|
||||
}
|
||||
|
||||
public boolean isDefaultSkip() {
|
||||
return defaultSkip;
|
||||
}
|
||||
|
||||
public void setDefaultSkip(boolean defaultSkip) {
|
||||
this.defaultSkip = defaultSkip;
|
||||
}
|
||||
}
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarOutputStream extends OutputStream {
|
||||
private final OutputStream out;
|
||||
private long bytesWritten;
|
||||
private long currentFileSize;
|
||||
private TarEntry currentEntry;
|
||||
|
||||
public TarOutputStream(OutputStream out) {
|
||||
this.out = out;
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
public TarOutputStream(final File fout) throws FileNotFoundException {
|
||||
this.out = new BufferedOutputStream(new FileOutputStream(fout));
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a file for writing.
|
||||
*/
|
||||
public TarOutputStream(final File fout, final boolean append) throws IOException {
|
||||
@SuppressWarnings("resource")
|
||||
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
|
||||
final long fileSize = fout.length();
|
||||
if (append && fileSize > TarConstants.EOF_BLOCK) {
|
||||
raf.seek(fileSize - TarConstants.EOF_BLOCK);
|
||||
}
|
||||
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the EOF record and closes the stream
|
||||
*
|
||||
* @see java.io.FilterOutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closeCurrentEntry();
|
||||
write( new byte[TarConstants.EOF_BLOCK] );
|
||||
out.close();
|
||||
}
|
||||
/**
|
||||
* Writes a byte to the stream and updates byte counters
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
out.write( b );
|
||||
bytesWritten += 1;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being written exceed the current entry size.
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null && !currentEntry.isDirectory()) {
|
||||
if (currentEntry.getSize() < currentFileSize + len) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
|
||||
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
|
||||
+ "] being written." );
|
||||
}
|
||||
}
|
||||
|
||||
out.write( b, off, len );
|
||||
|
||||
bytesWritten += len;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += len;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the next tar entry header on the stream
|
||||
*
|
||||
* @param entry
|
||||
* @throws IOException
|
||||
*/
|
||||
public void putNextEntry(TarEntry entry) throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
entry.writeEntryHeader( header );
|
||||
|
||||
write( header );
|
||||
|
||||
currentEntry = entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
|
||||
+ currentEntry.getSize() + "] has not been fully written." );
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0;
|
||||
|
||||
pad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pads the last content block
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void pad() throws IOException {
|
||||
if (bytesWritten > 0) {
|
||||
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
|
||||
|
||||
if (extra > 0) {
|
||||
write( new byte[TarConstants.DATA_BLOCK - extra] );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author Kamran
|
||||
*
|
||||
*/
|
||||
public class TarUtils {
|
||||
/**
|
||||
* Determines the tar file size of the given folder/file path
|
||||
*
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
public static long calculateTarSize(File path) {
|
||||
return tarSize(path) + TarConstants.EOF_BLOCK;
|
||||
}
|
||||
|
||||
private static long tarSize(File dir) {
|
||||
long size = 0;
|
||||
|
||||
if (dir.isFile()) {
|
||||
return entrySize(dir.length());
|
||||
} else {
|
||||
File[] subFiles = dir.listFiles();
|
||||
|
||||
if (subFiles != null && subFiles.length > 0) {
|
||||
for (File file : subFiles) {
|
||||
if (file.isFile()) {
|
||||
size += entrySize(file.length());
|
||||
} else {
|
||||
size += tarSize(file);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Empty folder header
|
||||
return TarConstants.HEADER_BLOCK;
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
private static long entrySize(long fileSize) {
|
||||
long size = 0;
|
||||
size += TarConstants.HEADER_BLOCK; // Header
|
||||
size += fileSize; // File size
|
||||
|
||||
long extra = size % TarConstants.DATA_BLOCK;
|
||||
|
||||
if (extra > 0) {
|
||||
size += (TarConstants.DATA_BLOCK - extra); // pad
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
public static String trim(String s, char c) {
|
||||
StringBuffer tmp = new StringBuffer(s);
|
||||
for (int i = 0; i < tmp.length(); i++) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = tmp.length() - 1; i >= 0; i--) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
return tmp.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
|
||||
package org.kivy.android;
|
||||
|
||||
import java.net.Socket;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import android.os.SystemClock;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import android.app.*;
|
||||
import android.content.*;
|
||||
import android.view.*;
|
||||
import android.view.SurfaceView;
|
||||
import android.app.Activity;
|
||||
import android.content.Intent;
|
||||
import android.util.Log;
|
||||
import android.widget.Toast;
|
||||
import android.os.AsyncTask;
|
||||
import android.os.Bundle;
|
||||
import android.os.PowerManager;
|
||||
import android.graphics.PixelFormat;
|
||||
import android.view.SurfaceHolder;
|
||||
import android.content.Context;
|
||||
import android.content.pm.ActivityInfo;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.pm.ApplicationInfo;
|
||||
import android.content.Intent;
|
||||
import android.widget.ImageView;
|
||||
import java.io.InputStream;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.Color;
|
||||
|
||||
import android.widget.AbsoluteLayout;
|
||||
|
||||
import android.webkit.WebViewClient;
|
||||
import android.webkit.WebView;
|
||||
|
||||
import org.kivy.android.PythonUtil;
|
||||
|
||||
import org.renpy.android.ResourceManager;
|
||||
import org.renpy.android.AssetExtract;
|
||||
|
||||
public class PythonActivity extends Activity {
|
||||
// This activity is modified from a mixture of the SDLActivity and
|
||||
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
|
||||
// specifics.
|
||||
|
||||
private static final String TAG = "PythonActivity";
|
||||
|
||||
public static PythonActivity mActivity = null;
|
||||
|
||||
/** If shared libraries (e.g. the native application) could not be loaded. */
|
||||
public static boolean mBrokenLibraries;
|
||||
|
||||
protected static Thread mPythonThread;
|
||||
|
||||
private ResourceManager resourceManager = null;
|
||||
private Bundle mMetaData = null;
|
||||
private PowerManager.WakeLock mWakeLock = null;
|
||||
|
||||
public String getAppRoot() {
|
||||
String app_root = getFilesDir().getAbsolutePath() + "/app";
|
||||
return app_root;
|
||||
}
|
||||
|
||||
public static void initialize() {
|
||||
// The static nature of the singleton and Android quirkiness force us to initialize everything here
|
||||
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
|
||||
mBrokenLibraries = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
Log.v(TAG, "My oncreate running");
|
||||
resourceManager = new ResourceManager(this);
|
||||
|
||||
Log.v(TAG, "Ready to unpack");
|
||||
File app_root_file = new File(getAppRoot());
|
||||
unpackData("private", app_root_file);
|
||||
|
||||
Log.v(TAG, "About to do super onCreate");
|
||||
super.onCreate(savedInstanceState);
|
||||
Log.v(TAG, "Did super onCreate");
|
||||
|
||||
this.mActivity = this;
|
||||
//this.showLoadingScreen();
|
||||
Log.v("Python", "Device: " + android.os.Build.DEVICE);
|
||||
Log.v("Python", "Model: " + android.os.Build.MODEL);
|
||||
|
||||
//Log.v(TAG, "Ready to unpack");
|
||||
//new UnpackFilesTask().execute(getAppRoot());
|
||||
|
||||
PythonActivity.initialize();
|
||||
|
||||
// Load shared libraries
|
||||
String errorMsgBrokenLib = "";
|
||||
try {
|
||||
loadLibraries();
|
||||
} catch(UnsatisfiedLinkError e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
} catch(Exception e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
}
|
||||
|
||||
if (mBrokenLibraries)
|
||||
{
|
||||
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
|
||||
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
|
||||
+ System.getProperty("line.separator")
|
||||
+ System.getProperty("line.separator")
|
||||
+ "Error: " + errorMsgBrokenLib);
|
||||
dlgAlert.setTitle("Python Error");
|
||||
dlgAlert.setPositiveButton("Exit",
|
||||
new DialogInterface.OnClickListener() {
|
||||
@Override
|
||||
public void onClick(DialogInterface dialog,int id) {
|
||||
// if this button is clicked, close current activity
|
||||
PythonActivity.mActivity.finish();
|
||||
}
|
||||
});
|
||||
dlgAlert.setCancelable(false);
|
||||
dlgAlert.create().show();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Set up the Python environment
|
||||
String app_root_dir = getAppRoot();
|
||||
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
|
||||
|
||||
Log.v(TAG, "Setting env vars for start.c and Python to use");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
|
||||
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
|
||||
|
||||
try {
|
||||
Log.v(TAG, "Access to our meta-data...");
|
||||
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
|
||||
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
|
||||
|
||||
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
|
||||
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
|
||||
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
|
||||
mActivity.mWakeLock.acquire();
|
||||
}
|
||||
} catch (PackageManager.NameNotFoundException e) {
|
||||
}
|
||||
|
||||
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
|
||||
PythonActivity.mPythonThread = pythonThread;
|
||||
pythonThread.start();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDestroy() {
|
||||
Log.i("Destroy", "end of app");
|
||||
super.onDestroy();
|
||||
|
||||
// make sure all child threads (python_thread) are stopped
|
||||
android.os.Process.killProcess(android.os.Process.myPid());
|
||||
}
|
||||
|
||||
public void loadLibraries() {
|
||||
String app_root = new String(getAppRoot());
|
||||
File app_root_file = new File(app_root);
|
||||
PythonUtil.loadLibraries(app_root_file);
|
||||
}
|
||||
|
||||
public void recursiveDelete(File f) {
|
||||
if (f.isDirectory()) {
|
||||
for (File r : f.listFiles()) {
|
||||
recursiveDelete(r);
|
||||
}
|
||||
}
|
||||
f.delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Show an error using a toast. (Only makes sense from non-UI
|
||||
* threads.)
|
||||
*/
|
||||
public void toastError(final String msg) {
|
||||
|
||||
final Activity thisActivity = this;
|
||||
|
||||
runOnUiThread(new Runnable () {
|
||||
public void run() {
|
||||
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
|
||||
}
|
||||
});
|
||||
|
||||
// Wait to show the error.
|
||||
synchronized (this) {
|
||||
try {
|
||||
this.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void unpackData(final String resource, File target) {
|
||||
|
||||
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
|
||||
|
||||
// The version of data in memory and on disk.
|
||||
String data_version = resourceManager.getString(resource + "_version");
|
||||
String disk_version = null;
|
||||
|
||||
Log.v(TAG, "Data version is " + data_version);
|
||||
|
||||
// If no version, no unpacking is necessary.
|
||||
if (data_version == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check the current disk version, if any.
|
||||
String filesDir = target.getAbsolutePath();
|
||||
String disk_version_fn = filesDir + "/" + resource + ".version";
|
||||
|
||||
try {
|
||||
byte buf[] = new byte[64];
|
||||
InputStream is = new FileInputStream(disk_version_fn);
|
||||
int len = is.read(buf);
|
||||
disk_version = new String(buf, 0, len);
|
||||
is.close();
|
||||
} catch (Exception e) {
|
||||
disk_version = "";
|
||||
}
|
||||
|
||||
// If the disk data is out of date, extract it and write the
|
||||
// version file.
|
||||
// if (! data_version.equals(disk_version)) {
|
||||
if (! data_version.equals(disk_version)) {
|
||||
Log.v(TAG, "Extracting " + resource + " assets.");
|
||||
|
||||
recursiveDelete(target);
|
||||
target.mkdirs();
|
||||
|
||||
AssetExtract ae = new AssetExtract(this);
|
||||
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
|
||||
toastError("Could not extract " + resource + " data.");
|
||||
}
|
||||
|
||||
try {
|
||||
// Write .nomedia.
|
||||
new File(target, ".nomedia").createNewFile();
|
||||
|
||||
// Write version file.
|
||||
FileOutputStream os = new FileOutputStream(disk_version_fn);
|
||||
os.write(data_version.getBytes());
|
||||
os.close();
|
||||
} catch (Exception e) {
|
||||
Log.w("python", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long lastBackClick = SystemClock.elapsedRealtime();
|
||||
@Override
|
||||
public boolean onKeyDown(int keyCode, KeyEvent event) {
|
||||
// If it wasn't the Back key or there's no web page history, bubble up to the default
|
||||
// system behavior (probably exit the activity)
|
||||
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
Toast.makeText(this, "Click again to close the app",
|
||||
Toast.LENGTH_LONG).show();
|
||||
return true;
|
||||
}
|
||||
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
return super.onKeyDown(keyCode, event);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onNewIntent
|
||||
//
|
||||
|
||||
public interface NewIntentListener {
|
||||
void onNewIntent(Intent intent);
|
||||
}
|
||||
|
||||
private List<NewIntentListener> newIntentListeners = null;
|
||||
|
||||
public void registerNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
|
||||
this.newIntentListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.newIntentListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onNewIntent(Intent intent) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.newIntentListeners ) {
|
||||
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
|
||||
while ( iterator.hasNext() ) {
|
||||
(iterator.next()).onNewIntent(intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onActivityResult
|
||||
//
|
||||
|
||||
public interface ActivityResultListener {
|
||||
void onActivityResult(int requestCode, int resultCode, Intent data);
|
||||
}
|
||||
|
||||
private List<ActivityResultListener> activityResultListeners = null;
|
||||
|
||||
public void registerActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
|
||||
this.activityResultListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.activityResultListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.activityResultListeners ) {
|
||||
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
|
||||
while ( iterator.hasNext() )
|
||||
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
|
||||
}
|
||||
}
|
||||
|
||||
public static void start_service(String serviceTitle, String serviceDescription,
|
||||
String pythonServiceArgument) {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
|
||||
String filesDirectory = argument;
|
||||
String app_root_dir = PythonActivity.mActivity.getAppRoot();
|
||||
serviceIntent.putExtra("androidPrivate", argument);
|
||||
serviceIntent.putExtra("androidArgument", app_root_dir);
|
||||
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
|
||||
serviceIntent.putExtra("pythonName", "python");
|
||||
serviceIntent.putExtra("pythonHome", app_root_dir);
|
||||
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
serviceIntent.putExtra("serviceTitle", serviceTitle);
|
||||
serviceIntent.putExtra("serviceDescription", serviceDescription);
|
||||
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
PythonActivity.mActivity.startService(serviceIntent);
|
||||
}
|
||||
|
||||
public static void stop_service() {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
PythonActivity.mActivity.stopService(serviceIntent);
|
||||
}
|
||||
|
||||
|
||||
public static native void nativeSetEnv(String j_name, String j_value);
|
||||
public static native int nativeInit(Object arguments);
|
||||
|
||||
}
|
||||
|
||||
|
||||
class PythonMain implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
PythonActivity.nativeInit(new String[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonEvent {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
private final Condition cond = lock.newCondition();
|
||||
private boolean flag = false;
|
||||
|
||||
public void set() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = true;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void wait_() throws InterruptedException {
|
||||
lock.lock();
|
||||
try {
|
||||
while (!flag) {
|
||||
cond.await();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = false;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonLock {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
|
||||
public void acquire() {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package org.renpy.android;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.IntentFilter;
|
||||
import android.hardware.Sensor;
|
||||
import android.hardware.SensorEvent;
|
||||
import android.hardware.SensorEventListener;
|
||||
import android.hardware.SensorManager;
|
||||
import android.net.ConnectivityManager;
|
||||
import android.net.NetworkInfo;
|
||||
import android.net.wifi.ScanResult;
|
||||
import android.net.wifi.WifiManager;
|
||||
import android.os.Vibrator;
|
||||
import android.view.View;
|
||||
|
||||
/**
|
||||
* Methods that are expected to be called via JNI, to access the device's
|
||||
* non-screen hardware. (For example, the vibration and accelerometer.)
|
||||
*/
|
||||
public class Hardware {
|
||||
|
||||
// The context.
|
||||
static Context context;
|
||||
static View view;
|
||||
|
||||
/**
|
||||
* Vibrate for s seconds.
|
||||
*/
|
||||
public static void vibrate(double s) {
|
||||
Vibrator v = (Vibrator) context
|
||||
.getSystemService(Context.VIBRATOR_SERVICE);
|
||||
if (v != null) {
|
||||
v.vibrate((int) (1000 * s));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an Overview of all Hardware Sensors of an Android Device
|
||||
*/
|
||||
public static String getHardwareSensors() {
|
||||
SensorManager sm = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
|
||||
|
||||
if (allSensors != null) {
|
||||
String resultString = "";
|
||||
for (Sensor s : allSensors) {
|
||||
resultString += String.format("Name=" + s.getName());
|
||||
resultString += String.format(",Vendor=" + s.getVendor());
|
||||
resultString += String.format(",Version=" + s.getVersion());
|
||||
resultString += String.format(",MaximumRange="
|
||||
+ s.getMaximumRange());
|
||||
// XXX MinDelay is not in the 2.2
|
||||
// resultString += String.format(",MinDelay=" +
|
||||
// s.getMinDelay());
|
||||
resultString += String.format(",Power=" + s.getPower());
|
||||
resultString += String.format(",Type=" + s.getType() + "\n");
|
||||
}
|
||||
return resultString;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
|
||||
* Magnetic Field Sensors
|
||||
*/
|
||||
public static class generic3AxisSensor implements SensorEventListener {
|
||||
private final SensorManager sSensorManager;
|
||||
private final Sensor sSensor;
|
||||
private final int sSensorType;
|
||||
SensorEvent sSensorEvent;
|
||||
|
||||
public generic3AxisSensor(int sensorType) {
|
||||
sSensorType = sensorType;
|
||||
sSensorManager = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
sSensor = sSensorManager.getDefaultSensor(sSensorType);
|
||||
}
|
||||
|
||||
public void onAccuracyChanged(Sensor sensor, int accuracy) {
|
||||
}
|
||||
|
||||
public void onSensorChanged(SensorEvent event) {
|
||||
sSensorEvent = event;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable the Sensor by registering/unregistering
|
||||
*/
|
||||
public void changeStatus(boolean enable) {
|
||||
if (enable) {
|
||||
sSensorManager.registerListener(this, sSensor,
|
||||
SensorManager.SENSOR_DELAY_NORMAL);
|
||||
} else {
|
||||
sSensorManager.unregisterListener(this, sSensor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the Sensor
|
||||
*/
|
||||
public float[] readSensor() {
|
||||
if (sSensorEvent != null) {
|
||||
return sSensorEvent.values;
|
||||
} else {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static generic3AxisSensor accelerometerSensor = null;
|
||||
public static generic3AxisSensor orientationSensor = null;
|
||||
public static generic3AxisSensor magneticFieldSensor = null;
|
||||
|
||||
/**
|
||||
* functions for backward compatibility reasons
|
||||
*/
|
||||
|
||||
public static void accelerometerEnable(boolean enable) {
|
||||
if (accelerometerSensor == null)
|
||||
accelerometerSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_ACCELEROMETER);
|
||||
accelerometerSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] accelerometerReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (accelerometerSensor == null)
|
||||
return rv;
|
||||
return (float[]) accelerometerSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void orientationSensorEnable(boolean enable) {
|
||||
if (orientationSensor == null)
|
||||
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
|
||||
orientationSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] orientationSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (orientationSensor == null)
|
||||
return rv;
|
||||
return (float[]) orientationSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void magneticFieldSensorEnable(boolean enable) {
|
||||
if (magneticFieldSensor == null)
|
||||
magneticFieldSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_MAGNETIC_FIELD);
|
||||
magneticFieldSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] magneticFieldSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (magneticFieldSensor == null)
|
||||
return rv;
|
||||
return (float[]) magneticFieldSensor.readSensor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan WiFi networks
|
||||
*/
|
||||
static List<ScanResult> latestResult;
|
||||
|
||||
public static void enableWifiScanner() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
|
||||
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
|
||||
// occurs
|
||||
WifiManager w = (WifiManager) c
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
latestResult = w.getScanResults(); // Returns a <list> of
|
||||
// scanResults
|
||||
}
|
||||
|
||||
}, i);
|
||||
|
||||
}
|
||||
|
||||
public static String scanWifi() {
|
||||
|
||||
// Now you can call this and it should execute the broadcastReceiver's
|
||||
// onReceive()
|
||||
WifiManager wm = (WifiManager) context
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
boolean a = wm.startScan();
|
||||
|
||||
if (latestResult != null) {
|
||||
|
||||
String latestResultString = "";
|
||||
for (ScanResult result : latestResult) {
|
||||
latestResultString += String.format("%s\t%s\t%d\n",
|
||||
result.SSID, result.BSSID, result.level);
|
||||
}
|
||||
|
||||
return latestResultString;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* network state
|
||||
*/
|
||||
|
||||
public static boolean network_state = false;
|
||||
|
||||
/**
|
||||
* Check network state directly
|
||||
*
|
||||
* (only one connection can be active at a given moment, detects all network
|
||||
* type)
|
||||
*
|
||||
*/
|
||||
public static boolean checkNetwork() {
|
||||
boolean state = false;
|
||||
final ConnectivityManager conMgr = (ConnectivityManager) context
|
||||
.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||
|
||||
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
|
||||
if (activeNetwork != null && activeNetwork.isConnected()) {
|
||||
state = true;
|
||||
} else {
|
||||
state = false;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* To recieve network state changes
|
||||
*/
|
||||
public static void registerNetworkCheck() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
network_state = checkNetwork();
|
||||
}
|
||||
|
||||
}, i);
|
||||
}
|
||||
|
||||
}
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,93 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="{{ args.package }}"
|
||||
android:versionCode="{{ args.numeric_version }}"
|
||||
android:versionName="{{ args.version }}"
|
||||
android:installLocation="auto">
|
||||
|
||||
<supports-screens
|
||||
android:smallScreens="true"
|
||||
android:normalScreens="true"
|
||||
android:largeScreens="true"
|
||||
android:anyDensity="true"
|
||||
{% if args.min_sdk_version >= 9 %}
|
||||
android:xlargeScreens="true"
|
||||
{% endif %}
|
||||
/>
|
||||
|
||||
<!-- Android 2.3.3 -->
|
||||
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
|
||||
|
||||
<!-- Set permissions -->
|
||||
{% for perm in args.permissions %}
|
||||
{% if '.' in perm %}
|
||||
<uses-permission android:name="{{ perm }}" />
|
||||
{% else %}
|
||||
<uses-permission android:name="android.permission.{{ perm }}" />
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% if args.wakelock %}
|
||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||
{% endif %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<uses-permission android:name="com.android.vending.BILLING" />
|
||||
{% endif %}
|
||||
|
||||
<!-- Create a Java class extending SDLActivity and place it in a
|
||||
directory under src matching the package, e.g.
|
||||
src/com/gamemaker/game/MyGame.java
|
||||
|
||||
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
|
||||
in the XML below.
|
||||
|
||||
An example Java class can be found in README-android.txt
|
||||
-->
|
||||
<application android:label="@string/app_name"
|
||||
android:icon="@drawable/icon"
|
||||
android:allowBackup="true"
|
||||
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
|
||||
android:hardwareAccelerated="true" >
|
||||
|
||||
{% for m in args.meta_data %}
|
||||
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
|
||||
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
|
||||
|
||||
<activity android:name="org.kivy.android.PythonActivity"
|
||||
android:label="@string/app_name"
|
||||
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
|
||||
>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
{%- if args.intent_filters -%}
|
||||
{{- args.intent_filters -}}
|
||||
{%- endif -%}
|
||||
</activity>
|
||||
|
||||
{% if service %}
|
||||
<service android:name="org.kivy.android.PythonService"
|
||||
android:process=":pythonservice" />
|
||||
{% endif %}
|
||||
{% for name in service_names %}
|
||||
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
|
||||
android:process=":service_{{ name }}" />
|
||||
{% endfor %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<service android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbilling" />
|
||||
<receiver android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbillingreceiver">
|
||||
<intent-filter>
|
||||
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
|
||||
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
|
||||
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
{% endif %}
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,74 @@
|
||||
package {{ args.package }};
|
||||
|
||||
import android.os.Binder;
|
||||
import android.os.IBinder;
|
||||
import android.content.Intent;
|
||||
import android.content.Context;
|
||||
import org.kivy.android.PythonService;
|
||||
|
||||
public class Service{{ name|capitalize }} extends PythonService {
|
||||
/**
|
||||
* Binder given to clients
|
||||
*/
|
||||
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
|
||||
|
||||
{% if sticky %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public int getStartType() {
|
||||
return START_STICKY;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% if foreground %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean getStartForeground() {
|
||||
return true;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
public static void start(Context ctx, String pythonServiceArgument) {
|
||||
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
intent.putExtra("androidPrivate", argument);
|
||||
intent.putExtra("androidArgument", argument);
|
||||
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
|
||||
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
|
||||
intent.putExtra("serviceDescription", "");
|
||||
intent.putExtra("pythonName", "{{ name }}");
|
||||
intent.putExtra("pythonHome", argument);
|
||||
intent.putExtra("androidUnpack", argument);
|
||||
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
|
||||
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
ctx.startService(intent);
|
||||
}
|
||||
|
||||
public static void stop(Context ctx) {
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
ctx.stopService(intent);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public IBinder onBind(Intent intent) {
|
||||
return mBinder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Class used for the client Binder. Because we know this service always
|
||||
* runs in the same process as its clients, we don't need to deal with IPC.
|
||||
*/
|
||||
public class Service{{ name|capitalize }}Binder extends Binder {
|
||||
Service{{ name|capitalize }} getService() {
|
||||
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
|
||||
return Service{{ name|capitalize }}.this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">{{ args.name }}</string>
|
||||
<string name="private_version">{{ private_version }}</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,68 @@
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from jnius import PythonJavaClass, autoclass, java_method
|
||||
JAVA_NAMESPACE = 'org.kivy.android'
|
||||
JNI_NAMESPACE = 'org/kivy/android'
|
||||
|
||||
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
|
||||
|
||||
_callbacks = {
|
||||
'on_new_intent': [],
|
||||
'on_activity_result': [],
|
||||
}
|
||||
|
||||
|
||||
class NewIntentListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback, **kwargs):
|
||||
super(NewIntentListener, self).__init__(**kwargs)
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(Landroid/content/Intent;)V')
|
||||
def onNewIntent(self, intent):
|
||||
self.callback(intent)
|
||||
|
||||
|
||||
class ActivityResultListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback):
|
||||
super(ActivityResultListener, self).__init__()
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(IILandroid/content/Intent;)V')
|
||||
def onActivityResult(self, requestCode, resultCode, intent):
|
||||
self.callback(requestCode, resultCode, intent)
|
||||
|
||||
|
||||
def bind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
elif event == 'on_new_intent':
|
||||
listener = NewIntentListener(callback)
|
||||
_activity.registerNewIntentListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
elif event == 'on_activity_result':
|
||||
listener = ActivityResultListener(callback)
|
||||
_activity.registerActivityResultListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
|
||||
|
||||
def unbind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
else:
|
||||
for listener in _callbacks[event][:]:
|
||||
if listener.callback == callback:
|
||||
_callbacks[event].remove(listener)
|
||||
if event == 'on_new_intent':
|
||||
_activity.unregisterNewIntentListener(listener)
|
||||
elif event == 'on_activity_result':
|
||||
_activity.unregisterActivityResultListener(listener)
|
||||
|
||||
|
||||
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
|
||||
@@ -0,0 +1,49 @@
|
||||
from os.path import join, dirname, abspath
|
||||
from pythonforandroid.toolchain import Bootstrap
|
||||
from setuptools import setup, find_packages
|
||||
from bootstrap import LBRYServiceBootstrap
|
||||
|
||||
|
||||
Bootstrap.bootstraps = {
|
||||
'lbry-service': LBRYServiceBootstrap()
|
||||
}
|
||||
|
||||
|
||||
setup(
|
||||
name='lbryservice',
|
||||
version='0.1',
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
description="Android Service for LBRY Network.",
|
||||
license='MIT',
|
||||
python_requires='>=3.7',
|
||||
packages=find_packages(),
|
||||
package_data={'service': ['*.py']},
|
||||
options={
|
||||
'apk': {
|
||||
'dist_name': 'lbry-service',
|
||||
'bootstrap': 'lbry-service',
|
||||
'package': 'io.lbry.service',
|
||||
'permissions': ['INTERNET'],
|
||||
'requirements': ','.join([
|
||||
# needed by aiohttp
|
||||
'multidict', 'yarl', 'async_timeout', 'chardet',
|
||||
# minimum needed by torba:
|
||||
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
|
||||
abspath(join(dirname(__file__), '..', '..', 'torba')),
|
||||
# minimum needed by lbrynet
|
||||
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
|
||||
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
|
||||
abspath(join(dirname(__file__), '..')),
|
||||
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
|
||||
]),
|
||||
'android-api': '26',
|
||||
'ndk-api': '21',
|
||||
'ndk-version': 'r17c',
|
||||
'arch': 'armeabi-v7a',
|
||||
'sdk-dir': '/home/lex/projects/android',
|
||||
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
|
||||
}
|
||||
}
|
||||
)
|
||||
+1531
-2481
File diff suppressed because it is too large.
Load diff
+4
-2
@@ -1409,7 +1409,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1954,6 +1954,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1978,6 +1979,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2584,4 +2586,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
+4
-2
@@ -1339,7 +1339,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1786,6 +1786,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1810,6 +1811,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2310,4 +2312,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.36.0rc3"
|
||||
__version__ = "0.31.0rc3"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+21
-31
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
@@ -18,11 +17,8 @@ from lbrynet.blob.writer import HashBlobWriter
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return len(char) == 1 and _hexmatch.match(char)
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
@@ -33,7 +29,7 @@ def is_valid_blobhash(blobhash: str) -> bool:
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
@@ -70,22 +66,18 @@ class BlobFile:
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if self.file_exists:
|
||||
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
|
||||
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.exception()
|
||||
error = finished.result()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
@@ -94,37 +86,37 @@ class BlobFile:
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
async def save_verified_blob(self, writer):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(verified_bytes)
|
||||
write_handle.write(writer.verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
if os.path.exists(self.file_path):
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
@@ -138,20 +130,18 @@ class BlobFile:
|
||||
"""
|
||||
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
return await self.loop.sendfile(writer.transport, handle)
|
||||
|
||||
def close(self):
|
||||
async def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
self.verified.clear()
|
||||
self.finished_writing.clear()
|
||||
self.length = None
|
||||
async def delete(self):
|
||||
await self.close()
|
||||
async with self.blob_write_lock:
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlite3 import IntegrityError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -30,21 +30,10 @@ class BlobFileManager:
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
|
||||
raw_blob_hashes = await self.get_all_verified_blobs()
|
||||
self.completed_blob_hashes.update(raw_blob_hashes)
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
@@ -63,28 +52,38 @@ class BlobFileManager:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
|
||||
await self.storage.add_completed_blob(blob.blob_hash)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
async def set_should_announce(self, blob_hash: str, should_announce: bool):
|
||||
now = self.loop.time()
|
||||
return await self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
async def get_all_verified_blobs(self) -> typing.List[str]:
|
||||
blob_hashes = await self.storage.get_all_blob_hashes()
|
||||
return self.check_completed_blobs(blob_hashes)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str]):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash:
|
||||
continue
|
||||
try:
|
||||
blob = self.get_blob(blob_hash)
|
||||
await blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
|
||||
for blob_hash in blob_hashes:
|
||||
self.delete_blob(blob_hash)
|
||||
|
||||
if delete_from_db:
|
||||
await self.storage.delete_blobs_from_db(blob_hashes)
|
||||
if blob_hash in self.blobs:
|
||||
del self.blobs[blob_hash]
|
||||
try:
|
||||
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
@@ -18,6 +18,7 @@ class HashBlobWriter:
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.buffer is not None:
|
||||
@@ -45,7 +46,7 @@ class HashBlobWriter:
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
self.finished.set_result(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
@@ -54,12 +55,15 @@ class HashBlobWriter:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.buffer is not None:
|
||||
|
||||
+81
-117
@@ -1,8 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
@@ -21,184 +19,150 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self.buf = b''
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
response = BlobResponse.deserialize(self.buf + data)
|
||||
if not response.responses and not self._response_fut.done():
|
||||
self.buf += data
|
||||
return
|
||||
else:
|
||||
self.buf = b''
|
||||
response = BlobResponse.deserialize(data)
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
|
||||
|
||||
def _write(self, data):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
def data_received(self, data):
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
self.transport.write(request.serialize())
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
return False, False
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("data rate rejected")
|
||||
return False, False
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
return False, True
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("incoming blob hash mismatch")
|
||||
return False, False
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("incoming blob unexpected length")
|
||||
return False, False
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
log.info(msg)
|
||||
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
|
||||
return self._blob_bytes_received, self.transport
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
return self._blob_bytes_received, self.close()
|
||||
except (InvalidBlobHashError, InvalidDataError):
|
||||
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
return False, False
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
def close(self):
|
||||
async def close(self):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.exception(e)
|
||||
return self._blob_bytes_received, self.transport
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
self.close()
|
||||
return self._blob_bytes_received, None
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
try:
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or blob.file_exists:
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return 0, None
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
+110
-122
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
@@ -13,132 +13,120 @@ if typing.TYPE_CHECKING:
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
def drain_into(a: list, b: list):
|
||||
while a:
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.time_since_last_blob = loop.time()
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'BlobFile'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
|
||||
# the safe net side is that any failure will reset the peer score, triggering the race back
|
||||
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
|
||||
# for peer, task in self.active_connections.items():
|
||||
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
|
||||
# return False
|
||||
return not (blob.get_is_verified() or blob.file_exists)
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
if blob.get_is_verified():
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport
|
||||
)
|
||||
if bytes_received == blob.get_length():
|
||||
self.time_since_last_blob = self.loop.time()
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.connections[peer] = transport
|
||||
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
|
||||
self.scores[peer] = rough_speed
|
||||
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
try:
|
||||
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
active_tasks = list(self.active_connections.values())
|
||||
try:
|
||||
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
finally:
|
||||
drain_tasks(tasks)
|
||||
|
||||
def cleanup_active(self):
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
if now - self.time_since_last_blob > 60.0:
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
|
||||
self.peer_queue.put_nowait(forgiven)
|
||||
for banned_peer in forgiven:
|
||||
self.ignored.pop(banned_peer)
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(self.peer_queue.get_nowait())
|
||||
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in batch:
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished(blob)
|
||||
self.cleanup_active()
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
|
||||
else:
|
||||
self.clearbanned()
|
||||
blob.close()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
finally:
|
||||
while self.active_connections:
|
||||
self.active_connections.popitem()[1].cancel()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
downloader.close()
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
@@ -146,26 +146,32 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
return None, response_msg
|
||||
break
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
|
||||
@@ -57,13 +57,8 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
@@ -77,7 +72,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
request = BlobRequest.deserialize(data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
@@ -100,7 +95,6 @@ class BlobServer:
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -108,7 +102,7 @@ class BlobServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
|
||||
+68
-74
@@ -16,11 +16,33 @@ log = logging.getLogger(__name__)
|
||||
NOT_SET = type(str('NOT_SET'), (object,), {})
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
API_ADDRESS = 'lbryapi'
|
||||
APP_NAME = 'LBRY'
|
||||
BLOBFILES_DIR = 'blobfiles'
|
||||
CRYPTSD_FILE_EXTENSION = '.cryptsd'
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
|
||||
LOG_FILE_NAME = 'lbrynet.log'
|
||||
LOG_POST_URL = 'https://lbry.io/log-upload'
|
||||
MAX_BLOB_REQUEST_SIZE = 64 * KB
|
||||
MAX_HANDSHAKE_SIZE = 64 * KB
|
||||
MAX_REQUEST_SIZE = 64 * KB
|
||||
MAX_RESPONSE_INFO_SIZE = 64 * KB
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
)
|
||||
HEADERS_FILE_SHA256_CHECKSUM = (
|
||||
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
|
||||
)
|
||||
@@ -449,7 +471,7 @@ class CLIConfig(BaseConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
# directories
|
||||
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
@@ -460,88 +482,60 @@ class Config(CLIConfig):
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
split_buckets_under_index = Integer(
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
# claims set to expire within this many blocks will be
|
||||
# automatically renewed after startup (if set to 0, renews
|
||||
# will not be made automatically)
|
||||
auto_renew_claim_height_delta = Integer("", 0)
|
||||
cache_time = Integer("", 150)
|
||||
data_rate = Float("points/megabyte", .0001)
|
||||
delete_blobs_on_remove = Toggle("", True)
|
||||
dht_node_port = Integer("", 4444)
|
||||
download_timeout = Float("", 30.0)
|
||||
blob_download_timeout = Float("", 20.0)
|
||||
peer_connect_timeout = Float("", 3.0)
|
||||
node_rpc_timeout = Float("", constants.rpc_timeout)
|
||||
is_generous_host = Toggle("", True)
|
||||
announce_head_blobs_only = Toggle("", True)
|
||||
concurrent_announcers = Integer("", 10)
|
||||
known_dht_nodes = Servers("", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
|
||||
# comment server uses a single string because requests are made to the /api resource
|
||||
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
max_connections_per_stream = Integer("", 5)
|
||||
seek_head_blob_first = Toggle("", True)
|
||||
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
|
||||
min_info_rate = Float("points/1000 infos", .02)
|
||||
min_valuable_hash_rate = Float("points/1000 infos", .05)
|
||||
min_valuable_info_rate = Float("points/1000 infos", .05)
|
||||
peer_port = Integer("", 3333)
|
||||
pointtrader_server = String("", 'http://127.0.0.1:2424')
|
||||
reflector_port = Integer("", 5566)
|
||||
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
reflect_uploads = Toggle("", True)
|
||||
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
|
||||
reflector_servers = Servers("", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
|
||||
sd_download_timeout = Integer("", 3)
|
||||
peer_search_timeout = Integer("", 60)
|
||||
use_upnp = Toggle("", True)
|
||||
use_keyring = Toggle("", False)
|
||||
blockchain_name = String("", 'lbrycrd_main')
|
||||
lbryum_servers = Servers("", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -19,8 +19,6 @@ class BlobAnnouncer:
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
@@ -41,7 +39,7 @@ class BlobAnnouncer:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
await self.storage.update_last_announced_blobs(announced, self.loop.time())
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
|
||||
@@ -6,7 +6,6 @@ hash_length = hash_class().digest_size
|
||||
hash_bits = hash_length * 8
|
||||
alpha = 5
|
||||
k = 8
|
||||
split_buckets_under_index = 1
|
||||
replacement_cache_size = 8
|
||||
rpc_timeout = 5.0
|
||||
rpc_attempts = 5
|
||||
@@ -16,7 +15,6 @@ refresh_interval = 3600 # 1 hour
|
||||
replicate_interval = refresh_interval
|
||||
data_expiration = 86400 # 24 hours
|
||||
token_secret_refresh_interval = 300 # 5 minutes
|
||||
maybe_ping_delay = 300 # 5 minutes
|
||||
check_refresh_interval = refresh_interval / 5
|
||||
max_datagram_size = 8192 # 8 KB
|
||||
rpc_id_length = 20
|
||||
|
||||
+41
-62
@@ -1,9 +1,9 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import socket
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -20,12 +20,10 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
split_buckets_under_index)
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -66,7 +64,7 @@ class Node:
|
||||
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
@@ -81,9 +79,10 @@ class Node:
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
log.info("Store to %i peers", len(peers))
|
||||
log.info(peers)
|
||||
for peer in peers:
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
@@ -117,50 +116,39 @@ class Node:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
known_node_addresses = known_node_addresses or []
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
info = await self.loop.getaddrinfo(
|
||||
host, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
if (info[0][4][0], port) not in known_node_addresses:
|
||||
known_node_addresses.append((info[0][4][0], port))
|
||||
futs = []
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
futs.append(peer.ping())
|
||||
if futs:
|
||||
await asyncio.wait(futs, loop=self.loop)
|
||||
|
||||
if known_node_addresses:
|
||||
while not self.protocol.routing_table.get_peers():
|
||||
success = False
|
||||
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await peer.ping()
|
||||
success = True
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
|
||||
round(self.protocol.rpc_timeout, 2))
|
||||
if success:
|
||||
break
|
||||
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
|
||||
# in the routing table with good peers who are near us
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
self.joined.set()
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
@@ -190,7 +178,10 @@ class Node:
|
||||
|
||||
async def _add_hashes_from_queue():
|
||||
while True:
|
||||
blob_hash = await hash_queue.get()
|
||||
try:
|
||||
blob_hash = await hash_queue.get()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_value_finder(
|
||||
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
|
||||
@@ -202,6 +193,10 @@ class Node:
|
||||
async with peer_generator as junction:
|
||||
yield junction
|
||||
await peer_generator.finished.wait()
|
||||
except asyncio.CancelledError:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
raise
|
||||
finally:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
@@ -219,29 +214,13 @@ class Node:
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(node_id, max_results=max_results,
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
log.info("peer search: %s", peers)
|
||||
accumulated.extend(peers)
|
||||
log.info("junction done")
|
||||
log.info("context done")
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait([
|
||||
peer for peer in peers
|
||||
if not (
|
||||
peer.address == self.protocol.external_ip
|
||||
and peer.tcp_port == self.protocol.peer_port
|
||||
)
|
||||
])
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue()
|
||||
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
+54
-36
@@ -3,8 +3,6 @@ import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import lru_cache
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
@@ -32,30 +30,37 @@ class PeerManager:
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
self._lock = asyncio.Lock(loop=loop)
|
||||
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
async def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
async with self._lock:
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
async def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
async with self._lock:
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
async def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
async with self._lock:
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
async def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
async with self._lock:
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
async def clear_token(self, node_id: bytes):
|
||||
async with self._lock:
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
async def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
async with self._lock:
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
@@ -65,37 +70,50 @@ class PeerManager:
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
|
||||
return self._node_id_mapping.get((address, udp_port))
|
||||
|
||||
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
return self._node_id_reverse_mapping.get(node_id)
|
||||
|
||||
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
async with self._lock:
|
||||
addr_tuple = self._node_id_reverse_mapping.get(node_id)
|
||||
if addr_tuple and addr_tuple in self._node_id_mapping:
|
||||
return addr_tuple
|
||||
|
||||
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
async with self._lock:
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
@lru_cache(maxsize=400)
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
async def prune(self):
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
async with self._lock:
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
|
||||
@@ -13,6 +13,11 @@ def cancel_task(task: typing.Optional[asyncio.Task]):
|
||||
task.cancel()
|
||||
|
||||
|
||||
def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
for task in tasks:
|
||||
cancel_task(task)
|
||||
|
||||
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
@@ -95,6 +100,6 @@ class AsyncGeneratorJunction:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -10,13 +10,11 @@ class Distance:
|
||||
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
if len(key_two) != constants.hash_length:
|
||||
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
@@ -92,8 +90,8 @@ class IterativeFinder:
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
|
||||
self.active: typing.List['KademliaPeer'] = []
|
||||
self.contacted: typing.List[typing.Tuple[str, int]] = []
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
@@ -101,12 +99,14 @@ class IterativeFinder:
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.running_probes: typing.List[asyncio.Task] = []
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
@@ -114,16 +114,9 @@ class IterativeFinder:
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
Called with a lock after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
@@ -136,7 +129,9 @@ class IterativeFinder:
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
if not self.closest_peer:
|
||||
return True
|
||||
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
@@ -146,28 +141,35 @@ class IterativeFinder:
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.add(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
async with self.lock:
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.append(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
addr_tuple = (contact_triple[1], contact_triple[2])
|
||||
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(
|
||||
contact_triple[0], contact_triple[1], contact_triple[2]
|
||||
)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
return
|
||||
except RemoteException:
|
||||
return
|
||||
@@ -179,35 +181,31 @@ class IterativeFinder:
|
||||
"""
|
||||
|
||||
added = 0
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.add((peer.address, peer.udp_port))
|
||||
async with self.lock:
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.append((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
def callback(_):
|
||||
if t and t in self.running_probes:
|
||||
self.running_probes.remove(t)
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.append(t)
|
||||
added += 1
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
@@ -217,40 +215,70 @@ class IterativeFinder:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
drain_tasks(self.running_probes)
|
||||
self.running = False
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def __aiter__(self):
|
||||
def search(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
|
||||
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
|
||||
peers = self.loop.create_task(self.iteration_queue.get())
|
||||
finished = self.loop.create_task(self.finished.wait())
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
if peers.done():
|
||||
return peers.result()
|
||||
raise StopAsyncIteration()
|
||||
except asyncio.CancelledError as error:
|
||||
err = error
|
||||
finally:
|
||||
if not finished.done() and not finished.cancelled():
|
||||
finished.cancel()
|
||||
if not peers.done() and not peers.cancelled():
|
||||
peers.cancel()
|
||||
if err:
|
||||
raise err
|
||||
|
||||
def __aiter__(self):
|
||||
self.search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
initial_results = self.get_initial_result()
|
||||
if initial_results:
|
||||
self.iteration_queue.put_nowait(initial_results)
|
||||
result = await self.next_queue_or_finished()
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
self.loop.call_soon(self.aclose)
|
||||
await self.aclose()
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
async def _aclose():
|
||||
async with self.lock:
|
||||
self.running = False
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
drain_tasks(self.tasks)
|
||||
drain_tasks(self.running_probes)
|
||||
while self.delayed_calls:
|
||||
timer = self.delayed_calls.pop()
|
||||
if timer:
|
||||
timer.cancel()
|
||||
|
||||
return asyncio.ensure_future(_aclose(), loop=self.loop)
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
@@ -267,26 +295,24 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
|
||||
def put_result(self, from_list: typing.List['KademliaPeer']):
|
||||
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.shortlist, finish=True)
|
||||
log.info("found")
|
||||
self.put_result(self.shortlist)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
@@ -297,10 +323,16 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.debug("max results")
|
||||
self.put_result(self.active, True)
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.info("max results")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
@@ -334,11 +366,14 @@ class IterativeValueFinder(IterativeFinder):
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
self.iteration_queue.put_nowait(None)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
+102
-103
@@ -77,23 +77,24 @@ class KademliaRPC:
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'contacts': self.find_node(rpc_contact, key)
|
||||
}
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
# get peers we have stored for this blob_exchange
|
||||
peers = [
|
||||
peer.compact_address_tcp()
|
||||
for peer in self.protocol.data_store.get_peers_for_blob(key)
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
@@ -179,7 +180,7 @@ class RemoteKademliaRPC:
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
@@ -187,46 +188,59 @@ class PingQueue:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
|
||||
self._loop = loop
|
||||
self._protocol = protocol
|
||||
self._enqueued_contacts: typing.List['KademliaPeer'] = []
|
||||
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
|
||||
self._process_task: asyncio.Task = None
|
||||
self._next_task: asyncio.Future = None
|
||||
self._next_timer: asyncio.TimerHandle = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
|
||||
now = self._loop.time()
|
||||
for peer in peers:
|
||||
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
|
||||
self._pending_contacts[peer] = delay + now
|
||||
async def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = constants.check_refresh_interval if delay is None else delay
|
||||
async with self._lock:
|
||||
for peer in peers:
|
||||
if delay and peer not in self._enqueued_contacts:
|
||||
self._pending_contacts[peer] = self._loop.time() + delay
|
||||
elif peer not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(peer)
|
||||
if peer in self._pending_contacts:
|
||||
del self._pending_contacts[peer]
|
||||
|
||||
def maybe_ping(self, peer: 'KademliaPeer'):
|
||||
async def ping_task():
|
||||
async def _process(self):
|
||||
async def _ping(p: 'KademliaPeer'):
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
await self._protocol.add_peer(peer)
|
||||
if self._protocol.peer_manager.peer_is_good(p):
|
||||
await self._protocol.add_peer(p)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
|
||||
task = self._loop.create_task(ping_task())
|
||||
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
|
||||
self._running_pings.add(task)
|
||||
|
||||
async def _process(self): # send up to 1 ping per second
|
||||
while True:
|
||||
enqueued = list(self._pending_contacts.keys())
|
||||
now = self._loop.time()
|
||||
for peer in enqueued:
|
||||
if self._pending_contacts[peer] <= now:
|
||||
del self._pending_contacts[peer]
|
||||
self.maybe_ping(peer)
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
tasks = []
|
||||
|
||||
async with self._lock:
|
||||
if self._enqueued_contacts or self._pending_contacts:
|
||||
now = self._loop.time()
|
||||
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
|
||||
for k in scheduled:
|
||||
del self._pending_contacts[k]
|
||||
if k not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(k)
|
||||
while self._enqueued_contacts:
|
||||
peer = self._enqueued_contacts.pop()
|
||||
tasks.append(self._loop.create_task(_ping(peer)))
|
||||
if tasks:
|
||||
await asyncio.wait(tasks, loop=self._loop)
|
||||
|
||||
f = self._loop.create_future()
|
||||
self._loop.call_later(1.0, lambda: None if f.done() else f.set_result(None))
|
||||
await f
|
||||
|
||||
def start(self):
|
||||
assert not self._running
|
||||
@@ -240,14 +254,17 @@ class PingQueue:
|
||||
if self._process_task:
|
||||
self._process_task.cancel()
|
||||
self._process_task = None
|
||||
while self._running_pings:
|
||||
self._running_pings.pop().cancel()
|
||||
if self._next_task:
|
||||
self._next_task.cancel()
|
||||
self._next_task = None
|
||||
if self._next_timer:
|
||||
self._next_timer.cancel()
|
||||
self._next_timer = None
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = 5.0):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
@@ -262,10 +279,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
@@ -299,14 +317,9 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
for p in self.routing_table.get_peers():
|
||||
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(p)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
@@ -400,32 +413,22 @@ class KademliaProtocol(DatagramProtocol):
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
await self.peer_manager.report_last_requested(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
await self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
await self.ping_queue.enqueue_maybe_ping(peer)
|
||||
elif is_good is True:
|
||||
await self.add_peer(peer)
|
||||
except ValueError as err:
|
||||
log.debug("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
await self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
|
||||
except Exception as err:
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
@@ -442,10 +445,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}, expected {peer.address}")
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
)
|
||||
return
|
||||
|
||||
peer.set_id(response_datagram.node_id)
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("node has our node id"))
|
||||
@@ -453,10 +457,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
peer.set_id(response_datagram.node_id)
|
||||
peer.update_udp_port(address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
await self.peer_manager.report_last_replied(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
@@ -473,23 +475,21 @@ class KademliaProtocol(DatagramProtocol):
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
)
|
||||
return
|
||||
|
||||
error_msg = f"" \
|
||||
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
|
||||
f"Args: {request.args}\n" \
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if 'Invalid token' in error_msg:
|
||||
log.debug(error_msg)
|
||||
elif error_datagram.response not in old_protocol_errors:
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
|
||||
# reject replies coming from a different address than what we sent our request to
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
log.error("node id mismatch in reply")
|
||||
remote_exception = TimeoutError(peer.node_id)
|
||||
df.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -522,14 +522,12 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
async with self._split_lock:
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
self.routing_table.remove_peer(peer)
|
||||
raise
|
||||
|
||||
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
@@ -556,29 +554,30 @@ class KademliaProtocol(DatagramProtocol):
|
||||
if message.rpc_id in self.sent_messages:
|
||||
self.sent_messages.pop(message.rpc_id)
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
async with self.lock:
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
@@ -602,7 +601,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
@@ -610,7 +609,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
await self.peer_manager.clear_token(peer.node_id)
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
@@ -29,7 +29,6 @@ class KBucket:
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
self._distance_to_self = Distance(node_id)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
@@ -49,14 +48,7 @@ class KBucket:
|
||||
self.peers.remove(peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
p = self.peers[i]
|
||||
if p.node_id == peer.node_id:
|
||||
self.peers.remove(p)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < constants.k:
|
||||
elif len(self.peers) < constants.k:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
@@ -138,7 +130,7 @@ class KBucket:
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
return self.range_min <= self._distance_to_self(key) < self.range_max
|
||||
return self.range_min <= int.from_bytes(key, 'big') < self.range_max
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.peers)
|
||||
@@ -163,12 +155,10 @@ class TreeRoutingTable:
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
|
||||
@@ -180,7 +170,7 @@ class TreeRoutingTable:
|
||||
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if bucket_index < self._split_buckets_under_index:
|
||||
if self.buckets[bucket_index].key_in_range(self._parent_node_id):
|
||||
return True
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
@@ -193,6 +183,8 @@ class TreeRoutingTable:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
if key in exclude:
|
||||
exclude.remove(key)
|
||||
count = count or constants.k
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
@@ -246,15 +238,11 @@ class TreeRoutingTable:
|
||||
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
|
||||
return random_id.to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
).to_bytes(constants.hash_length, 'big')
|
||||
return int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
@@ -281,8 +269,6 @@ class TreeRoutingTable:
|
||||
old_bucket.remove_peer(contact)
|
||||
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
|
||||
if not to_pop:
|
||||
return
|
||||
|
||||
@@ -65,36 +65,46 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
|
||||
@classmethod
|
||||
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
elif not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
|
||||
|
||||
@classmethod
|
||||
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f"invalid port: {port}")
|
||||
if len(token) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid token length: {len(token)}")
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
store_args = [blob_hash, token, port, from_node_id, 0]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
|
||||
|
||||
@classmethod
|
||||
def make_find_node(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
|
||||
|
||||
@classmethod
|
||||
def make_find_value(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
|
||||
|
||||
|
||||
@@ -138,6 +148,8 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if not isinstance(primitive, dict):
|
||||
raise ValueError("invalid datagram type")
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
@@ -151,19 +163,14 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
)
|
||||
|
||||
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
return compact_ip
|
||||
def make_compact_ip(address: str):
|
||||
return reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
|
||||
|
||||
def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
compact_ip = make_compact_ip(address)
|
||||
if not 0 < port < 65536:
|
||||
if not 0 <= port <= 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
@@ -171,8 +178,4 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
address = "{}.{}.{}.{}".format(*compact_address[:4])
|
||||
port = int.from_bytes(compact_address[4:6], 'big')
|
||||
node_id = compact_address[6:]
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return node_id, address, port
|
||||
+2
-1
@@ -28,7 +28,8 @@ class DownloadTimeoutError(Exception):
|
||||
|
||||
class DownloadDataTimeout(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
|
||||
super().__init__('Failed to download data blobs for sd hash '
|
||||
'{} within timeout'.format(download))
|
||||
self.download = download
|
||||
|
||||
|
||||
|
||||
+2
-29
@@ -1,7 +1,4 @@
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -219,19 +216,12 @@ def get_argument_parser():
|
||||
return main
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
@@ -248,23 +238,17 @@ def main(argv=None):
|
||||
)
|
||||
file_handler.setFormatter(default_formatter)
|
||||
log.addHandler(file_handler)
|
||||
logging.getLogger('torba').addHandler(file_handler)
|
||||
|
||||
if not args.quiet:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
log.addHandler(handler)
|
||||
logging.getLogger('torba').addHandler(handler)
|
||||
logging.getLogger('torba').setLevel(logging.INFO)
|
||||
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
loop.set_debug(True)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
if conf.share_usage_data:
|
||||
@@ -273,21 +257,10 @@ def main(argv=None):
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
daemon = Daemon(conf)
|
||||
started = False
|
||||
def __exit():
|
||||
if started:
|
||||
daemon.stop_event.set()
|
||||
else:
|
||||
raise GracefulExit()
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, __exit)
|
||||
loop.add_signal_handler(signal.SIGTERM, __exit)
|
||||
except NotImplementedError:
|
||||
pass # Not implemented on Windows
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(daemon.start())
|
||||
started = True
|
||||
loop.run_until_complete(daemon.stop_event.wait())
|
||||
loop.run_forever()
|
||||
except (GracefulExit, KeyboardInterrupt):
|
||||
pass
|
||||
finally:
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import socket
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
import base58
|
||||
@@ -12,9 +14,11 @@ from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
@@ -22,8 +26,8 @@ from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.wallet import LbryWalletManager
|
||||
from lbrynet.wallet import Network
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -50,6 +54,25 @@ async def gather_dict(tasks: dict):
|
||||
)))
|
||||
|
||||
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
|
||||
info = await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
return info[0][4][0]
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -93,7 +116,7 @@ class DatabaseComponent(Component):
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
@@ -134,31 +157,40 @@ class HeadersComponent(Component):
|
||||
}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
def collector(d, h_file):
|
||||
h_file.write(d)
|
||||
local_size = float(h_file.tell())
|
||||
final_size = float(final_size_after_download)
|
||||
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
|
||||
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
if response.content_length % HEADER_SIZE != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
async with aiohttp.request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
got_406 = response.status == 406 # our file is bigger
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while True:
|
||||
chunk = await response.content.read(512)
|
||||
if not chunk:
|
||||
break
|
||||
fd.write(chunk)
|
||||
self._headers_progress_percent = math.ceil(
|
||||
float(fd.tell()) / float(final_size_after_download) * 100
|
||||
)
|
||||
log.info("fetched headers from s3, now verifying integrity after download.")
|
||||
self._check_header_file_integrity()
|
||||
if got_406:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
# should have something to download and a final length divisible by the header size
|
||||
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
|
||||
s3_height = (final_size_after_download / HEADER_SIZE) - 1
|
||||
local_height = self.local_header_file_height()
|
||||
if s3_height > local_height:
|
||||
data = await response.read()
|
||||
|
||||
if local_header_size:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
with open(self.headers_file, "a+b") as headers_file:
|
||||
collector(data, headers_file)
|
||||
else:
|
||||
with open(self.headers_file, "wb") as headers_file:
|
||||
collector(data, headers_file)
|
||||
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.",
|
||||
s3_height)
|
||||
self._check_header_file_integrity()
|
||||
else:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
else:
|
||||
log.error("invalid size for headers from s3")
|
||||
|
||||
def local_header_file_height(self):
|
||||
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
|
||||
@@ -251,20 +283,23 @@ class WalletComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
remote_height = await self.wallet_manager.ledger.network.get_server_height()
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.use_encryption,
|
||||
'is_encrypted': self.wallet_manager.wallet.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
await self.wallet_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
@@ -296,7 +331,9 @@ class BlobComponent(Component):
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
self.blob_manager.stop()
|
||||
while self.blob_manager and self.blob_manager.blobs:
|
||||
_, blob = self.blob_manager.blobs.popitem()
|
||||
await blob.close()
|
||||
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
@@ -322,7 +359,7 @@ class DHTComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
@@ -339,12 +376,12 @@ class DHTComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -352,15 +389,13 @@ class DHTComponent(Component):
|
||||
asyncio.get_event_loop(),
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
internal_udp_port=self.conf.dht_node_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index
|
||||
peer_port=self.external_peer_port
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
|
||||
)
|
||||
log.info("Started the dht")
|
||||
|
||||
@@ -384,7 +419,7 @@ class HashAnnouncerComponent(Component):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
self.hash_announcer.start(self.conf.concurrent_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
async def stop(self):
|
||||
@@ -427,13 +462,17 @@ class StreamManagerComponent(Component):
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
|
||||
self.conf.peer_connect_timeout, [
|
||||
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
|
||||
for url, port in self.conf.reflector_servers
|
||||
], self.conf.reflector_servers
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
await self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -453,10 +492,10 @@ class PeerProtocolServerComponent(Component):
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
self.blob_server.start_server(peer_port, interface='0.0.0.0')
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
@@ -469,8 +508,8 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
@@ -509,7 +548,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -560,7 +599,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip = await get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
@@ -575,9 +614,7 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if self.component_manager.analytics_manager:
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
@@ -603,7 +640,7 @@ class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
Component.__init__(self, component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
|
||||
+1035
-1516
File diff suppressed because it is too large.
Load diff
+124
-116
@@ -1,14 +1,12 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
import aiohttp
|
||||
import typing
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
@@ -21,137 +19,102 @@ HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
|
||||
TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: int, tried_peers_count: int,
|
||||
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_download_duration: typing.Optional[float] = None,
|
||||
error: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
"error": error,
|
||||
'name': name,
|
||||
"outpoint": outpoint,
|
||||
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
"tried_peers_count": tried_peers_count,
|
||||
|
||||
"sd_blob_hash": sd_hash,
|
||||
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
|
||||
|
||||
"head_blob_hash": head_blob_hash,
|
||||
"head_blob_length": head_blob_length,
|
||||
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
|
||||
}
|
||||
|
||||
|
||||
def _make_context(platform):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
class Manager:
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
async def start(self):
|
||||
def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, data: typing.Dict):
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + '/track',
|
||||
'url': self.url + endpoint,
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with utils.aiohttp_request(**request_kwargs) as response:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
|
||||
async def track(self, event: typing.Dict):
|
||||
async def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post(event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
@@ -170,33 +133,20 @@ class AnalyticsManager:
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
|
||||
total_duration: typing.Optional[float], download_id: str,
|
||||
name: str, outpoint: str, found_peers_count: int,
|
||||
tried_peers_count: int, added_fixed_peers: bool,
|
||||
fixed_peers_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_duration: typing.Optional[int] = None,
|
||||
error: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
|
||||
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
|
||||
)))
|
||||
|
||||
async def send_download_finished(self, download_id, name, sd_hash):
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
await self.track(
|
||||
self._event(
|
||||
DOWNLOAD_FINISHED, {
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
)
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
@@ -209,11 +159,69 @@ class AnalyticsManager:
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
@@ -1,31 +0,0 @@
|
||||
import logging
|
||||
import aiohttp
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpc_body(method: str, rpc_id: any, **params) -> dict:
|
||||
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
clean = params.pop('clean', True)
|
||||
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
|
||||
if clean:
|
||||
if 'error' in response:
|
||||
return response['error']
|
||||
return response['result']
|
||||
else:
|
||||
return response
|
||||
|
||||
|
||||
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
complete = []
|
||||
batch_size = max(batch_size, 50)
|
||||
for i in range(0, len(calls), batch_size):
|
||||
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
|
||||
complete += await response.json()
|
||||
if clean:
|
||||
complete = [body['result'] if 'result' in body else None for body in complete]
|
||||
return complete
|
||||
@@ -1,11 +1,10 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.utils import aiohttp_request
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -53,8 +52,8 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
async def _make_request(self) -> str:
|
||||
async with aiohttp_request('get', self.url, params=self.params) as response:
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
@@ -220,19 +219,19 @@ class ExchangeRateManager:
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
if from_currency == to_currency:
|
||||
return amount
|
||||
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return amount * Decimal(market.rate.spot)
|
||||
return amount * market.rate.spot
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot))
|
||||
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -4,117 +4,25 @@ from binascii import hexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from ecdsa import BadSignatureError
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.wallet.ledger import MainNetLedger, Account
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.extras.wallet import MainNetLedger
|
||||
from lbrynet.extras.wallet.transaction import Transaction, Output
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_txo_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'nout': "position in the transaction",
|
||||
'height': "block where transaction was recorded",
|
||||
'amount': "value of the txo as a decimal",
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
}
|
||||
|
||||
|
||||
def encode_tx_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'height': "block where transaction was recorded",
|
||||
'inputs': [encode_txo_doc()],
|
||||
'outputs': [encode_txo_doc()],
|
||||
'total_input': "sum of inputs as a decimal",
|
||||
'total_output': "sum of outputs, sans fee, as a decimal",
|
||||
'total_fee': "fee amount",
|
||||
'hex': "entire transaction encoded in hex",
|
||||
}
|
||||
|
||||
|
||||
def encode_account_doc():
|
||||
return {
|
||||
'id': 'account_id',
|
||||
'is_default': 'this account is used by default',
|
||||
'ledger': 'name of crypto currency and network',
|
||||
'name': 'optional account name',
|
||||
'seed': 'human friendly words from which account can be recreated',
|
||||
'encrypted': 'if account is encrypted',
|
||||
'private_key': 'extended private key',
|
||||
'public_key': 'extended public key',
|
||||
'address_generator': 'settings for generating addresses',
|
||||
'modified_on': 'date of last modification to account settings'
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'completed': '(bool) true if download is completed',
|
||||
'file_name': '(str) name of file',
|
||||
'download_directory': '(str) download directory',
|
||||
'points_paid': '(float) credit paid to download file',
|
||||
'stopped': '(bool) true if download is stopped',
|
||||
'stream_hash': '(str) stream hash of file',
|
||||
'stream_name': '(str) stream name',
|
||||
'suggested_file_name': '(str) suggested file name',
|
||||
'sd_hash': '(str) sd hash of file',
|
||||
'download_path': '(str) download path of file',
|
||||
'mime_type': '(str) mime type of file',
|
||||
'key': '(str) key attached to file',
|
||||
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
|
||||
'total_bytes': '(int) file upper bound size in bytes',
|
||||
'written_bytes': '(int) written size in bytes',
|
||||
'blobs_completed': '(int) number of fully downloaded blobs',
|
||||
'blobs_in_stream': '(int) total blobs on stream',
|
||||
'blobs_remaining': '(int) total blobs remaining to download',
|
||||
'status': '(str) downloader status',
|
||||
'claim_id': '(str) None if claim is not found else the claim id',
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
|
||||
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -135,15 +43,15 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo, check_signature=True):
|
||||
def encode_output(self, txo):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = self.ledger.headers.height
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'height': tx_height,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
@@ -151,64 +59,46 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'create'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'payment'
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
})
|
||||
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
output['value'] = txo.claim
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = {
|
||||
'name': txo.channel.claim_name,
|
||||
'claim_id': txo.channel.claim_id,
|
||||
'value': txo.channel.claim
|
||||
}
|
||||
if check_signature and txo.claim.is_signed:
|
||||
output['is_channel_signature_valid'] = False
|
||||
claim = txo.claim
|
||||
output['value'] = claim.claim_dict
|
||||
if claim.has_signature:
|
||||
output['valid_signature'] = None
|
||||
if txo.channel is not None:
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
try:
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
except BadSignatureError:
|
||||
pass
|
||||
output['valid_signature'] = False
|
||||
except ValueError:
|
||||
log.exception(
|
||||
'txo.id: %s, txo.channel.id:%s, output: %s',
|
||||
txo.id, txo.channel.id, output
|
||||
)
|
||||
output['valid_signature'] = False
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'basic'
|
||||
|
||||
return output
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
|
||||
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
|
||||
def encode_account(self, account):
|
||||
result = account.to_dict()
|
||||
result['id'] = account.id
|
||||
result.pop('certificates', None)
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def encode_file(managed_stream):
|
||||
return managed_stream.as_dict()
|
||||
|
||||
@staticmethod
|
||||
def encode_claim(claim):
|
||||
return getattr(claim, claim.claim_type).to_dict()
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
@@ -50,7 +51,7 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -260,7 +260,7 @@ def do_migration(conf):
|
||||
(
|
||||
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
|
||||
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
|
||||
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
)
|
||||
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
|
||||
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import os
|
||||
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
'.a': 'application/octet-stream',
|
||||
'.ai': 'application/postscript',
|
||||
'.aif': 'audio/x-aiff',
|
||||
'.aifc': 'audio/x-aiff',
|
||||
'.aiff': 'audio/x-aiff',
|
||||
'.au': 'audio/basic',
|
||||
'.avi': 'video/x-msvideo',
|
||||
'.bat': 'text/plain',
|
||||
'.bcpio': 'application/x-bcpio',
|
||||
'.bin': 'application/octet-stream',
|
||||
'.bmp': 'image/bmp',
|
||||
'.c': 'text/plain',
|
||||
'.cdf': 'application/x-netcdf',
|
||||
'.cpio': 'application/x-cpio',
|
||||
'.csh': 'application/x-csh',
|
||||
'.css': 'text/css',
|
||||
'.csv': 'text/csv',
|
||||
'.dll': 'application/octet-stream',
|
||||
'.doc': 'application/msword',
|
||||
'.dot': 'application/msword',
|
||||
'.dvi': 'application/x-dvi',
|
||||
'.eml': 'message/rfc822',
|
||||
'.eps': 'application/postscript',
|
||||
'.epub': 'application/epub+zip',
|
||||
'.etx': 'text/x-setext',
|
||||
'.exe': 'application/octet-stream',
|
||||
'.gif': 'image/gif',
|
||||
'.gtar': 'application/x-gtar',
|
||||
'.h': 'text/plain',
|
||||
'.hdf': 'application/x-hdf',
|
||||
'.htm': 'text/html',
|
||||
'.html': 'text/html',
|
||||
'.ico': 'image/vnd.microsoft.icon',
|
||||
'.ief': 'image/ief',
|
||||
'.iges': 'model/iges',
|
||||
'.jpe': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.js': 'application/javascript',
|
||||
'.json': 'application/json',
|
||||
'.ksh': 'text/plain',
|
||||
'.latex': 'application/x-latex',
|
||||
'.m1v': 'video/mpeg',
|
||||
'.m3u': 'application/vnd.apple.mpegurl',
|
||||
'.m3u8': 'application/vnd.apple.mpegurl',
|
||||
'.man': 'application/x-troff-man',
|
||||
'.markdown': 'text/markdown',
|
||||
'.md': 'text/markdown',
|
||||
'.me': 'application/x-troff-me',
|
||||
'.mht': 'message/rfc822',
|
||||
'.mhtml': 'message/rfc822',
|
||||
'.mif': 'application/x-mif',
|
||||
'.mov': 'video/quicktime',
|
||||
'.movie': 'video/x-sgi-movie',
|
||||
'.mp2': 'audio/mpeg',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.mp4': 'video/mp4',
|
||||
'.mpa': 'video/mpeg',
|
||||
'.mpe': 'video/mpeg',
|
||||
'.mpeg': 'video/mpeg',
|
||||
'.mpg': 'video/mpeg',
|
||||
'.ms': 'application/x-troff-ms',
|
||||
'.nc': 'application/x-netcdf',
|
||||
'.nws': 'message/rfc822',
|
||||
'.o': 'application/octet-stream',
|
||||
'.obj': 'application/octet-stream',
|
||||
'.oda': 'application/oda',
|
||||
'.p12': 'application/x-pkcs12',
|
||||
'.p7c': 'application/pkcs7-mime',
|
||||
'.pbm': 'image/x-portable-bitmap',
|
||||
'.pdf': 'application/pdf',
|
||||
'.pfx': 'application/x-pkcs12',
|
||||
'.pgm': 'image/x-portable-graymap',
|
||||
'.pl': 'text/plain',
|
||||
'.png': 'image/png',
|
||||
'.pnm': 'image/x-portable-anymap',
|
||||
'.pot': 'application/vnd.ms-powerpoint',
|
||||
'.ppa': 'application/vnd.ms-powerpoint',
|
||||
'.ppm': 'image/x-portable-pixmap',
|
||||
'.pps': 'application/vnd.ms-powerpoint',
|
||||
'.ppt': 'application/vnd.ms-powerpoint',
|
||||
'.ps': 'application/postscript',
|
||||
'.pwz': 'application/vnd.ms-powerpoint',
|
||||
'.py': 'text/x-python',
|
||||
'.pyc': 'application/x-python-code',
|
||||
'.pyo': 'application/x-python-code',
|
||||
'.qt': 'video/quicktime',
|
||||
'.ra': 'audio/x-pn-realaudio',
|
||||
'.ram': 'application/x-pn-realaudio',
|
||||
'.ras': 'image/x-cmu-raster',
|
||||
'.rdf': 'application/xml',
|
||||
'.rgb': 'image/x-rgb',
|
||||
'.roff': 'application/x-troff',
|
||||
'.rtx': 'text/richtext',
|
||||
'.sgm': 'text/x-sgml',
|
||||
'.sgml': 'text/x-sgml',
|
||||
'.sh': 'application/x-sh',
|
||||
'.shar': 'application/x-shar',
|
||||
'.snd': 'audio/basic',
|
||||
'.so': 'application/octet-stream',
|
||||
'.src': 'application/x-wais-source',
|
||||
'.stl': 'model/stl',
|
||||
'.sv4cpio': 'application/x-sv4cpio',
|
||||
'.sv4crc': 'application/x-sv4crc',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.swf': 'application/x-shockwave-flash',
|
||||
'.t': 'application/x-troff',
|
||||
'.tar': 'application/x-tar',
|
||||
'.tcl': 'application/x-tcl',
|
||||
'.tex': 'application/x-tex',
|
||||
'.texi': 'application/x-texinfo',
|
||||
'.texinfo': 'application/x-texinfo',
|
||||
'.tif': 'image/tiff',
|
||||
'.tiff': 'image/tiff',
|
||||
'.tr': 'application/x-troff',
|
||||
'.tsv': 'text/tab-separated-values',
|
||||
'.txt': 'text/plain',
|
||||
'.ustar': 'application/x-ustar',
|
||||
'.vcf': 'text/x-vcard',
|
||||
'.wav': 'audio/x-wav',
|
||||
'.webm': 'video/webm',
|
||||
'.wiz': 'application/msword',
|
||||
'.wsdl': 'application/xml',
|
||||
'.xbm': 'image/x-xbitmap',
|
||||
'.xlb': 'application/vnd.ms-excel',
|
||||
'.xls': 'application/vnd.ms-excel',
|
||||
'.xml': 'text/xml',
|
||||
'.xpdl': 'application/xml',
|
||||
'.xpm': 'image/x-xpixmap',
|
||||
'.xsl': 'application/xml',
|
||||
'.xwd': 'image/x-xwindowdump',
|
||||
'.zip': 'application/zip',
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.mid': 'audio/midi',
|
||||
'.midi': 'audio/midi',
|
||||
'.pct': 'image/pict',
|
||||
'.pic': 'image/pict',
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': 'video/x-ms-wmv'
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
+147
-226
@@ -1,14 +1,13 @@
|
||||
import os
|
||||
import logging
|
||||
import sqlite3
|
||||
import typing
|
||||
import asyncio
|
||||
import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.dht.constants import data_expiration
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
|
||||
@@ -38,9 +37,7 @@ class StoredStreamClaim:
|
||||
self.claim_name = name
|
||||
self.amount = amount
|
||||
self.height = height
|
||||
self.claim: typing.Optional[Claim] = None if not serialized else Claim.from_bytes(
|
||||
binascii.unhexlify(serialized)
|
||||
)
|
||||
self.claim: typing.Optional[ClaimDict] = None if not serialized else smart_decode(serialized)
|
||||
self.claim_address = address
|
||||
self.claim_sequence = claim_sequence
|
||||
self.channel_claim_id = channel_claim_id
|
||||
@@ -107,104 +104,14 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
|
||||
return StoredStreamClaim(*claim_fields)
|
||||
|
||||
|
||||
def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
for start_index in range(0, len(parameters), batch_size):
|
||||
current_batch = parameters[start_index:start_index+batch_size]
|
||||
def _batched_select(transaction, query, parameters):
|
||||
for start_index in range(0, len(parameters), 900):
|
||||
current_batch = parameters[start_index:start_index+900]
|
||||
bind = "({})".format(','.join(['?'] * len(current_batch)))
|
||||
for result in transaction.execute(query.format(bind), current_batch):
|
||||
yield result
|
||||
|
||||
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", [
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
|
||||
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
signed_claims[claim.channel_claim_id] = []
|
||||
signed_claims[claim.channel_claim_id].append(claim)
|
||||
files.append(
|
||||
{
|
||||
"rowid": rowid,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name, # hex
|
||||
"download_directory": download_dir, # hex
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim
|
||||
}
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
list(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
|
||||
data_payment_rate: float, status: str) -> int:
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
)
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
@@ -277,12 +184,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
|
||||
def __init__(self, conf: Config, path, loop=None):
|
||||
super().__init__(path)
|
||||
self.conf = conf
|
||||
self.content_claim_callbacks = {}
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
self.time_getter = time_getter or time.time
|
||||
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
@@ -296,16 +202,15 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash: str, length: int):
|
||||
def _add_blob(transaction: sqlite3.Connection):
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
transaction.execute(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
|
||||
)
|
||||
return self.db.run(_add_blob)
|
||||
def add_completed_blob(self, blob_hash: str):
|
||||
log.debug("Adding a completed blob. blob_hash=%s", blob_hash)
|
||||
return self.db.execute("update blob set status='finished' where blob.blob_hash=?", (blob_hash, ))
|
||||
|
||||
def set_should_announce(self, blob_hash: str, next_announce_time: int, should_announce: int):
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
|
||||
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
|
||||
)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -342,9 +247,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select count(*) from blob where status='finished'"
|
||||
)
|
||||
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str], last_announced: float):
|
||||
def _update_last_announced_blobs(transaction: sqlite3.Connection):
|
||||
last_announced = self.time_getter()
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
@@ -355,7 +259,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = int(self.time_getter())
|
||||
now = self.loop.time()
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
@@ -370,23 +274,21 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = int(self.time_getter())
|
||||
if self.conf.announce_head_and_sd_only:
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished'",
|
||||
(timestamp,)
|
||||
)
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
"and next_announce_time<? and status='finished'", (timestamp,)
|
||||
)
|
||||
return [b[0] for b in r.fetchall()]
|
||||
blobs = [b[0] for b in r.fetchall()]
|
||||
return blobs
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
@@ -394,31 +296,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
|
||||
)
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
return self.db.run(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
to_update = [
|
||||
(blob_hash, )
|
||||
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
|
||||
if blob_hash not in blob_files
|
||||
]
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
to_update
|
||||
)
|
||||
return {
|
||||
blob_hash
|
||||
for blob_hash, in _batched_select(
|
||||
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
|
||||
list(blob_files)
|
||||
)
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
@@ -431,20 +313,42 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
|
||||
return streams is not None
|
||||
|
||||
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
|
||||
return self.run_and_return_one_or_none(
|
||||
"select rowid from file where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
return self.db.run(store_stream, sd_blob, descriptor)
|
||||
def _store_stream(transaction: sqlite3.Connection):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]:
|
||||
return self.db.run(_store_stream)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False):
|
||||
def _get_blobs_for_stream(transaction):
|
||||
crypt_blob_infos = []
|
||||
stream_blobs = transaction.execute(
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=? "
|
||||
"order by position asc", (stream_hash, )
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=?", (stream_hash, )
|
||||
).fetchall()
|
||||
if only_completed:
|
||||
lengths = transaction.execute(
|
||||
@@ -466,8 +370,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for blob_hash, position, iv in stream_blobs:
|
||||
blob_length = blob_length_dict.get(blob_hash, 0)
|
||||
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
|
||||
if not blob_hash:
|
||||
break
|
||||
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
|
||||
return crypt_blob_infos
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
@@ -482,75 +385,83 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
def _delete_stream(transaction: sqlite3.Connection):
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, ))
|
||||
transaction.executemany("delete from blob where blob_hash=?",
|
||||
[(blob.blob_hash, ) for blob in descriptor.blobs[:-1]])
|
||||
return self.db.run(_delete_stream)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory,
|
||||
data_payment_rate) -> typing.Awaitable[int]:
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
return self.save_published_file(
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
|
||||
status="running"
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished") -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
status="finished"):
|
||||
return self.db.execute(
|
||||
"insert into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
|
||||
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name) -> typing.Dict:
|
||||
return {
|
||||
"row_id": rowid,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name,
|
||||
"download_directory": download_dir,
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name,
|
||||
"suggested_file_name": suggested_file_name
|
||||
}
|
||||
|
||||
def _get_all_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
file_infos = list(map(lambda a: _lbry_file_dict(*a), transaction.execute(
|
||||
"select file.rowid, file.*, stream.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash"
|
||||
).fetchall()))
|
||||
stream_hashes = [file_info['stream_hash'] for file_info in file_infos]
|
||||
claim_infos = get_claims_from_stream_hashes(transaction, stream_hashes)
|
||||
for index in range(len(file_infos)): # pylint: disable=consider-using-enumerate
|
||||
file_infos[index]['claim'] = claim_infos.get(file_infos[index]['stream_hash'])
|
||||
return file_infos
|
||||
|
||||
results = await self.db.run(_get_all_files)
|
||||
if results:
|
||||
return results
|
||||
return []
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
log.info("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
|
||||
for descriptor, sd_blob in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped')
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
[(download_dir, stream_hash) for stream_hash in stream_hashes]
|
||||
)
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
# # # # # # # # # support functions # # # # # # # # #
|
||||
|
||||
def save_supports(self, claim_id_to_supports: dict):
|
||||
def save_supports(self, claim_id, supports):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
transaction.execute("delete from support where claim_id=?", (claim_id,))
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -578,7 +489,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
async def save_claims(self, claim_infos):
|
||||
claim_id_to_supports = {}
|
||||
support_callbacks = []
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
@@ -591,12 +502,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
height = claim_info['height']
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
certificate_id = claim_info['value'].signing_channel_id
|
||||
try:
|
||||
source_hash = claim_info['value'].stream.source.sd_hash
|
||||
except (AttributeError, ValueError):
|
||||
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
|
||||
except AttributeError:
|
||||
certificate_id = None
|
||||
try:
|
||||
if claim_info['value'].get('stream', {}).get('source', {}).get('sourceType') == "lbry_sd_hash":
|
||||
source_hash = claim_info['value'].get('stream', {}).get('source', {}).get('source')
|
||||
else:
|
||||
source_hash = None
|
||||
except AttributeError:
|
||||
source_hash = None
|
||||
serialized = binascii.hexlify(claim_info['value'].to_bytes())
|
||||
serialized = claim_info.get('hex') or binascii.hexlify(
|
||||
smart_decode(claim_info['value']).serialized).decode()
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
@@ -604,7 +522,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
claim_id_to_supports[claim_id] = claim_info['supports']
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -616,12 +534,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
stream_hash = stream_hash[0]
|
||||
known_outpoint = transaction.execute(
|
||||
"select claim_outpoint from content_claim where stream_hash=?", (stream_hash,)
|
||||
).fetchone()
|
||||
)
|
||||
known_claim_id = transaction.execute(
|
||||
"select claim_id from claim "
|
||||
"inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint "
|
||||
"where c3.stream_hash=?", (stream_hash,)
|
||||
).fetchone()
|
||||
)
|
||||
if not known_claim_id:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
@@ -634,20 +552,23 @@ class SQLiteStorage(SQLiteMixin):
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = {}
|
||||
to_save = []
|
||||
for info in claim_infos:
|
||||
if 'value' in info:
|
||||
if info['value']:
|
||||
to_save[info['claim_id']] = info
|
||||
to_save.append(info)
|
||||
else:
|
||||
for key in ('certificate', 'claim'):
|
||||
if info.get(key, {}).get('value'):
|
||||
to_save[info[key]['claim_id']] = info[key]
|
||||
return self.save_claims(to_save.values())
|
||||
if 'certificate' in info and info['certificate']['value']:
|
||||
to_save.append(info['certificate'])
|
||||
if 'claim' in info and info['claim']['value']:
|
||||
to_save.append(info['claim'])
|
||||
return self.save_claims(to_save)
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
@@ -657,7 +578,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
).fetchone()
|
||||
if not claim_info:
|
||||
raise Exception("claim not found")
|
||||
new_claim_id, claim = claim_info[0], Claim.from_bytes(binascii.unhexlify(claim_info[1]))
|
||||
new_claim_id, claim = claim_info[0], ClaimDict.deserialize(binascii.unhexlify(claim_info[1]))
|
||||
|
||||
# certificate claims should not be in the content_claim table
|
||||
if not claim.is_stream:
|
||||
@@ -670,7 +591,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if not known_sd_hash:
|
||||
raise Exception("stream not found")
|
||||
# check the claim contains the same sd hash
|
||||
if known_sd_hash[0] != claim.stream.source.sd_hash:
|
||||
if known_sd_hash[0].encode() != claim.source_hash:
|
||||
raise Exception("stream mismatch")
|
||||
|
||||
# if there is a current claim associated to the file, check that the new claim is an update to it
|
||||
@@ -761,7 +682,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
)
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
@@ -773,5 +694,5 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
int(self.time_getter()) - 86400
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
)
|
||||
@@ -2,6 +2,7 @@ import platform
|
||||
import os
|
||||
import logging.handlers
|
||||
|
||||
from lbrynet.schema import __version__ as schema_version
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -15,6 +16,7 @@ def get_platform() -> dict:
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"lbryschema_version": schema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
__node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
|
||||
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbrynet.extras.wallet.manager import LbryWalletManager
|
||||
from lbrynet.extras.wallet.network import Network
|
||||
@@ -1,51 +1,38 @@
|
||||
import json
|
||||
import logging
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
from string import hexdigits
|
||||
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
from torba.client.basetransaction import TXORef
|
||||
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.signer import SECP256k1, get_signer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def validate_claim_id(claim_id):
|
||||
if not len(claim_id) == 40:
|
||||
raise Exception("Incorrect claimid length: %i" % len(claim_id))
|
||||
if isinstance(claim_id, bytes):
|
||||
claim_id = claim_id.decode('utf-8')
|
||||
if set(claim_id).difference(hexdigits):
|
||||
raise Exception("Claim id is not hex encoded")
|
||||
def generate_certificate():
|
||||
secp256k1_private_key = get_signer(SECP256k1).generate().private_key.to_pem()
|
||||
return ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1), secp256k1_private_key
|
||||
|
||||
|
||||
class Account(BaseAccount):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.channel_keys = {}
|
||||
self.certificates = {}
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256(json.dumps(self.to_dict(False)).encode())
|
||||
for cert in sorted(self.channel_keys.keys()):
|
||||
h.update(cert.encode())
|
||||
return h.digest()
|
||||
def add_certificate_private_key(self, ref: TXORef, private_key):
|
||||
assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
|
||||
self.certificates[ref.id] = private_key
|
||||
|
||||
def apply(self, d: dict):
|
||||
super().apply(d)
|
||||
self.channel_keys.update(d.get('certificates', {}))
|
||||
|
||||
def add_channel_private_key(self, ref: TXORef, private_key):
|
||||
assert ref.id not in self.channel_keys, 'Trying to add a duplicate channel private key.'
|
||||
self.channel_keys[ref.id] = private_key
|
||||
|
||||
def get_channel_private_key(self, ref: TXORef):
|
||||
return self.channel_keys.get(ref.id)
|
||||
def get_certificate_private_key(self, ref: TXORef):
|
||||
return self.certificates.get(ref.id)
|
||||
|
||||
async def maybe_migrate_certificates(self):
|
||||
if not self.channel_keys:
|
||||
if not self.certificates:
|
||||
return
|
||||
|
||||
addresses = {}
|
||||
@@ -59,7 +46,7 @@ class Account(BaseAccount):
|
||||
}
|
||||
double_hex_encoded_to_pop = []
|
||||
|
||||
for maybe_claim_id in list(self.channel_keys):
|
||||
for maybe_claim_id in list(self.certificates):
|
||||
if ':' not in maybe_claim_id:
|
||||
try:
|
||||
validate_claim_id(maybe_claim_id)
|
||||
@@ -71,7 +58,7 @@ class Account(BaseAccount):
|
||||
maybe_claim_id_bytes = maybe_claim_id_bytes.encode()
|
||||
decoded_double_hex = binascii.unhexlify(maybe_claim_id_bytes).decode()
|
||||
validate_claim_id(decoded_double_hex)
|
||||
if decoded_double_hex in self.channel_keys:
|
||||
if decoded_double_hex in self.certificates:
|
||||
log.warning("don't know how to migrate certificate %s", decoded_double_hex)
|
||||
else:
|
||||
log.info("claim id was double hex encoded, fixing it")
|
||||
@@ -80,9 +67,9 @@ class Account(BaseAccount):
|
||||
continue
|
||||
|
||||
for double_encoded_claim_id, correct_claim_id in double_hex_encoded_to_pop:
|
||||
self.channel_keys[correct_claim_id] = self.channel_keys.pop(double_encoded_claim_id)
|
||||
self.certificates[correct_claim_id] = self.certificates.pop(double_encoded_claim_id)
|
||||
|
||||
for maybe_claim_id in list(self.channel_keys):
|
||||
for maybe_claim_id in list(self.certificates):
|
||||
results['total'] += 1
|
||||
if ':' not in maybe_claim_id:
|
||||
try:
|
||||
@@ -117,8 +104,8 @@ class Account(BaseAccount):
|
||||
.format(maybe_claim_id)
|
||||
)
|
||||
tx_nout = '{txid}:{nout}'.format(**claim)
|
||||
self.channel_keys[tx_nout] = self.channel_keys[maybe_claim_id]
|
||||
del self.channel_keys[maybe_claim_id]
|
||||
self.certificates[tx_nout] = self.certificates[maybe_claim_id]
|
||||
del self.certificates[maybe_claim_id]
|
||||
log.info(
|
||||
"Migrated certificate with claim_id '%s' ('%s') to a new look up key %s.",
|
||||
maybe_claim_id, txo.script.values['claim_name'], tx_nout
|
||||
@@ -178,7 +165,7 @@ class Account(BaseAccount):
|
||||
return super().get_balance(confirmations, **constraints)
|
||||
|
||||
@classmethod
|
||||
def get_private_key_from_seed(cls, ledger, seed: str, password: str):
|
||||
def get_private_key_from_seed(cls, ledger: 'baseledger.BaseLedger', seed: str, password: str):
|
||||
return super().get_private_key_from_seed(
|
||||
ledger, seed, password or 'lbryum'
|
||||
)
|
||||
@@ -186,30 +173,35 @@ class Account(BaseAccount):
|
||||
@classmethod
|
||||
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
|
||||
account = super().from_dict(ledger, wallet, d)
|
||||
account.channel_keys = d.get('certificates', {})
|
||||
account.certificates = d.get('certificates', {})
|
||||
return account
|
||||
|
||||
def to_dict(self, include_channel_keys=True):
|
||||
def to_dict(self):
|
||||
d = super().to_dict()
|
||||
if include_channel_keys:
|
||||
d['certificates'] = self.channel_keys
|
||||
d['certificates'] = self.certificates
|
||||
return d
|
||||
|
||||
async def get_details(self, **kwargs):
|
||||
details = await super().get_details(**kwargs)
|
||||
details['certificates'] = len(self.channel_keys)
|
||||
details['certificates'] = len(self.certificates)
|
||||
return details
|
||||
|
||||
def get_claim(self, claim_id=None, txid=None, nout=None):
|
||||
if claim_id is not None:
|
||||
return self.ledger.db.get_claims(account=self, claim_id=claim_id)
|
||||
elif txid is not None and nout is not None:
|
||||
return self.ledger.db.get_claims(**{'account': self, 'txo.txid': txid, 'txo.position': nout})
|
||||
|
||||
@staticmethod
|
||||
def constraint_spending_utxos(constraints):
|
||||
def constraint_utxos_sans_claims(constraints):
|
||||
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constraint_spending_utxos(constraints)
|
||||
self.constraint_utxos_sans_claims(constraints)
|
||||
return super().get_utxos(**constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
self.constraint_spending_utxos(constraints)
|
||||
self.constraint_utxos_sans_claims(constraints)
|
||||
return super().get_utxo_count(**constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
@@ -218,23 +210,32 @@ class Account(BaseAccount):
|
||||
def get_claim_count(self, **constraints):
|
||||
return self.ledger.db.get_claim_count(account=self, **constraints)
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
return self.ledger.db.get_streams(account=self, **constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
return self.ledger.db.get_stream_count(account=self, **constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
return self.ledger.db.get_channels(account=self, **constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.ledger.db.get_channel_count(account=self, **constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
return self.ledger.db.get_supports(account=self, **constraints)
|
||||
async def send_to_addresses(self, amount, addresses, broadcast=False):
|
||||
tx_class = self.ledger.transaction_class
|
||||
tx = await tx_class.create(
|
||||
inputs=[],
|
||||
outputs=[
|
||||
tx_class.output_class.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))
|
||||
for address in addresses
|
||||
],
|
||||
funding_accounts=[self],
|
||||
change_account=self
|
||||
)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
return self.ledger.db.get_support_count(account=self, **constraints)
|
||||
if broadcast:
|
||||
await self.ledger.broadcast(tx)
|
||||
else:
|
||||
await self.ledger.release_outputs(
|
||||
[txi.txo_ref.txo for txi in tx.inputs]
|
||||
)
|
||||
|
||||
return tx
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -74,10 +74,9 @@ def verify_proof(proof, root_hash, name):
|
||||
if 'txhash' in proof and 'nOut' in proof:
|
||||
if not verified_value:
|
||||
raise InvalidProofError("mismatch between proof claim and outcome")
|
||||
target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
|
||||
if 'txhash' in proof and 'nOut' in proof:
|
||||
if name != target:
|
||||
if name != reverse_computed_name[::-1]:
|
||||
raise InvalidProofError("name did not match proof")
|
||||
if not name.startswith(target):
|
||||
if not name.startswith(reverse_computed_name[::-1]):
|
||||
raise InvalidProofError("name fragment does not match proof")
|
||||
return True
|
||||
@@ -1,9 +1,5 @@
|
||||
from typing import List
|
||||
|
||||
from torba.client.basedatabase import BaseDatabase
|
||||
|
||||
from lbrynet.wallet.transaction import Output
|
||||
|
||||
|
||||
class WalletDatabase(BaseDatabase):
|
||||
|
||||
@@ -52,7 +48,7 @@ class WalletDatabase(BaseDatabase):
|
||||
row['claim_name'] = txo.claim_name
|
||||
return row
|
||||
|
||||
async def get_txos(self, **constraints) -> List[Output]:
|
||||
async def get_txos(self, **constraints):
|
||||
my_account = constraints.get('my_account', constraints.get('account', None))
|
||||
|
||||
txos = await super().get_txos(**constraints)
|
||||
@@ -60,10 +56,10 @@ class WalletDatabase(BaseDatabase):
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and my_account is not None:
|
||||
txo.private_key = my_account.get_channel_private_key(txo.ref)
|
||||
if 'publisherSignature' in txo.claim_dict:
|
||||
channel_ids.add(txo.claim_dict['publisherSignature']['certificateId'])
|
||||
if txo.claim_name.startswith('@') and my_account is not None:
|
||||
txo.private_key = my_account.get_certificate_private_key(txo.ref)
|
||||
|
||||
if channel_ids:
|
||||
channels = {
|
||||
@@ -75,35 +71,23 @@ class WalletDatabase(BaseDatabase):
|
||||
}
|
||||
for txo in txos:
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_id, None)
|
||||
if 'publisherSignature' in txo.claim_dict:
|
||||
txo.channel = channels.get(txo.claim_dict['publisherSignature']['certificateId'])
|
||||
|
||||
return txos
|
||||
|
||||
@staticmethod
|
||||
def constrain_claims(constraints):
|
||||
constraints['claim_type__any'] = {'is_claim': 1, 'is_update': 1}
|
||||
constraints['claim_type__any'] = {'is_claim': 1, 'is_update': 1, 'is_support': 1}
|
||||
|
||||
async def get_claims(self, **constraints) -> List[Output]:
|
||||
def get_claims(self, **constraints):
|
||||
self.constrain_claims(constraints)
|
||||
return await self.get_utxos(**constraints)
|
||||
return self.get_utxos(**constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
self.constrain_claims(constraints)
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
@staticmethod
|
||||
def constrain_streams(constraints):
|
||||
if 'claim_name' not in constraints or 'claim_id' not in constraints:
|
||||
constraints['claim_name__not_like'] = '@%'
|
||||
|
||||
def get_streams(self, **constraints):
|
||||
self.constrain_streams(constraints)
|
||||
return self.get_claims(**constraints)
|
||||
|
||||
def get_stream_count(self, **constraints):
|
||||
self.constrain_streams(constraints)
|
||||
return self.get_claim_count(**constraints)
|
||||
|
||||
@staticmethod
|
||||
def constrain_channels(constraints):
|
||||
if 'claim_name' not in constraints or 'claim_id' not in constraints:
|
||||
@@ -117,17 +101,22 @@ class WalletDatabase(BaseDatabase):
|
||||
self.constrain_channels(constraints)
|
||||
return self.get_claim_count(**constraints)
|
||||
|
||||
@staticmethod
|
||||
def constrain_supports(constraints):
|
||||
constraints['is_support'] = 1
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
self.constrain_supports(constraints)
|
||||
return self.get_utxos(**constraints)
|
||||
|
||||
def get_support_count(self, **constraints):
|
||||
self.constrain_supports(constraints)
|
||||
return self.get_utxo_count(**constraints)
|
||||
async def get_certificates(self, private_key_accounts, exclude_without_key=False, **constraints):
|
||||
channels = await self.get_channels(**constraints)
|
||||
certificates = []
|
||||
if private_key_accounts is not None:
|
||||
for channel in channels:
|
||||
if not channel.has_private_key:
|
||||
private_key = None
|
||||
for account in private_key_accounts:
|
||||
private_key = account.get_certificate_private_key(channel.ref)
|
||||
if private_key is not None:
|
||||
break
|
||||
if private_key is None and exclude_without_key:
|
||||
continue
|
||||
channel.private_key = private_key
|
||||
certificates.append(channel)
|
||||
return certificates
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute(
|
||||
File renamed without changes.
File renamed without changes.
@@ -2,14 +2,17 @@ import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from torba.client.baseledger import BaseLedger
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.wallet.resolve import Resolver
|
||||
from lbrynet.wallet.account import Account
|
||||
from lbrynet.wallet.network import Network
|
||||
from lbrynet.wallet.database import WalletDatabase
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
from lbrynet.wallet.header import Headers, UnvalidatedHeaders
|
||||
from lbrynet.schema.error import URIParseError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.wallet.resolve import Resolver
|
||||
from lbrynet.extras.wallet.account import Account
|
||||
from lbrynet.extras.wallet.network import Network
|
||||
from lbrynet.extras.wallet.database import WalletDatabase
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.extras.wallet.header import Headers, UnvalidatedHeaders
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -28,8 +31,6 @@ class MainNetLedger(BaseLedger):
|
||||
network_class = Network
|
||||
transaction_class = Transaction
|
||||
|
||||
db: WalletDatabase
|
||||
|
||||
secret_prefix = bytes((0x1c,))
|
||||
pubkey_address_prefix = bytes((0x55,))
|
||||
script_address_prefix = bytes((0x7a,))
|
||||
@@ -47,20 +48,38 @@ class MainNetLedger(BaseLedger):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||
self.resolver = Resolver(self)
|
||||
|
||||
def resolve(self, page, page_size, *uris):
|
||||
return self.resolver.resolve(page, page_size, *uris)
|
||||
@property
|
||||
def resolver(self):
|
||||
return Resolver(self.headers.claim_trie_root, self.headers.height, self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address, network=self.network)
|
||||
|
||||
async def resolve(self, page, page_size, *uris):
|
||||
for uri in uris:
|
||||
try:
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.claim_id:
|
||||
validate_claim_id(parsed_uri.claim_id)
|
||||
except URIParseError as err:
|
||||
return {'error': err.args[0]}
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
try:
|
||||
resolver = self.resolver
|
||||
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
|
||||
return await resolver._handle_resolutions(resolutions, uris, page, page_size)
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
|
||||
async def get_claim_by_claim_id(self, claim_id):
|
||||
result = (await self.network.get_claims_by_ids(claim_id)).pop(claim_id, {})
|
||||
return await self.resolver.parse_and_validate_claim_result(result)
|
||||
return await self.resolver.get_certificate_and_validate_result(result)
|
||||
|
||||
async def get_claim_by_outpoint(self, txid, nout):
|
||||
claims = (await self.network.get_claims_in_tx(txid)) or []
|
||||
for claim in claims:
|
||||
if claim['nout'] == nout:
|
||||
return await self.resolver.parse_and_validate_claim_result(claim)
|
||||
return await self.resolver.get_certificate_and_validate_result(claim)
|
||||
return 'claim not found'
|
||||
|
||||
async def start(self):
|
||||
@@ -79,7 +98,7 @@ class MainNetLedger(BaseLedger):
|
||||
log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), "
|
||||
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
|
||||
account.id, balance, total_receiving, account.receiving.gap, total_change, account.change.gap,
|
||||
channel_count, len(account.channel_keys), claim_count)
|
||||
channel_count, len(account.certificates), claim_count)
|
||||
|
||||
|
||||
class TestNetLedger(MainNetLedger):
|
||||
@@ -4,20 +4,43 @@ import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from lbrynet.schema.schema import SECP256k1
|
||||
from torba.client.basemanager import BaseWalletManager
|
||||
from torba.rpc.jsonrpc import CodeMessageError
|
||||
|
||||
from lbrynet.wallet.ledger import MainNetLedger
|
||||
from lbrynet.wallet.account import BaseAccount
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
from lbrynet.wallet.database import WalletDatabase
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger
|
||||
from lbrynet.extras.wallet.account import BaseAccount, generate_certificate
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.extras.wallet.database import WalletDatabase
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReservedPoints:
|
||||
def __init__(self, identifier, amount):
|
||||
self.identifier = identifier
|
||||
self.amount = amount
|
||||
|
||||
|
||||
class BackwardsCompatibleNetwork:
|
||||
def __init__(self, manager):
|
||||
self.manager = manager
|
||||
|
||||
def get_local_height(self):
|
||||
for ledger in self.manager.ledgers.values():
|
||||
assert isinstance(ledger, MainNetLedger)
|
||||
return ledger.headers.height
|
||||
|
||||
def get_server_height(self):
|
||||
return self.get_local_height()
|
||||
|
||||
|
||||
class LbryWalletManager(BaseWalletManager):
|
||||
|
||||
@property
|
||||
@@ -28,10 +51,22 @@ class LbryWalletManager(BaseWalletManager):
|
||||
def db(self) -> WalletDatabase:
|
||||
return self.ledger.db
|
||||
|
||||
@property
|
||||
def wallet(self):
|
||||
return self
|
||||
|
||||
@property
|
||||
def network(self):
|
||||
return BackwardsCompatibleNetwork(self)
|
||||
|
||||
@property
|
||||
def use_encryption(self):
|
||||
return self.default_account.serialize_encrypted
|
||||
|
||||
@property
|
||||
def is_first_run(self):
|
||||
return True
|
||||
|
||||
@property
|
||||
def is_wallet_unlocked(self):
|
||||
return not self.default_account.encrypted
|
||||
@@ -122,7 +157,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
return receiving_addresses, change_addresses
|
||||
|
||||
@classmethod
|
||||
async def from_lbrynet_config(cls, settings):
|
||||
async def from_lbrynet_config(cls, settings, db):
|
||||
|
||||
ledger_id = {
|
||||
'lbrycrd_main': 'lbc_mainnet',
|
||||
@@ -187,12 +222,65 @@ class LbryWalletManager(BaseWalletManager):
|
||||
def get_unused_address(self):
|
||||
return self.default_account.receiving.get_or_create_usable_address()
|
||||
|
||||
def get_new_address(self):
|
||||
return self.get_unused_address()
|
||||
|
||||
def reserve_points(self, address, amount: int):
|
||||
# TODO: check if we have enough to cover amount
|
||||
return ReservedPoints(address, amount)
|
||||
|
||||
async def send_amount_to_address(self, amount: int, destination_address: bytes, account=None):
|
||||
account = account or self.default_account
|
||||
tx = await Transaction.pay(amount, destination_address, [account], account)
|
||||
await account.ledger.broadcast(tx)
|
||||
return tx
|
||||
|
||||
async def send_claim_to_address(self, claim_id: str, destination_address: str, amount: Optional[int],
|
||||
account=None):
|
||||
account = account or self.default_account
|
||||
claims = await account.get_claims(
|
||||
claim_name_type__any={'is_claim': 1, 'is_update': 1}, # exclude is_supports
|
||||
claim_id=claim_id
|
||||
)
|
||||
if not claims:
|
||||
raise NameError(f"Claim not found: {claim_id}")
|
||||
if not amount:
|
||||
amount = claims[0].get_estimator(self.ledger).effective_amount
|
||||
tx = await Transaction.update(
|
||||
claims[0], ClaimDict.deserialize(claims[0].script.values['claim']), amount,
|
||||
destination_address.encode(), [account], account
|
||||
)
|
||||
await self.ledger.broadcast(tx)
|
||||
return tx
|
||||
|
||||
def send_points_to_address(self, reserved: ReservedPoints, amount: int, account=None):
|
||||
destination_address: bytes = reserved.identifier.encode('latin1')
|
||||
return self.send_amount_to_address(amount, destination_address, account)
|
||||
|
||||
async def resolve(self, *uris, **kwargs):
|
||||
page = kwargs.get('page', 0)
|
||||
page_size = kwargs.get('page_size', 10)
|
||||
check_cache = kwargs.get('check_cache', False) # TODO: put caching back (was force_refresh parameter)
|
||||
ledger: MainNetLedger = self.default_account.ledger
|
||||
results = await ledger.resolve(page, page_size, *uris)
|
||||
if 'error' not in results:
|
||||
await self.old_db.save_claims_for_resolve([
|
||||
value for value in results.values() if 'error' not in value
|
||||
])
|
||||
return results
|
||||
|
||||
async def get_claims_for_name(self, name: str):
|
||||
response = await self.ledger.network.get_claims_for_name(name)
|
||||
resolutions = await self.resolve(*(f"{claim['name']}#{claim['claim_id']}" for claim in response['claims']))
|
||||
response['claims'] = [value.get('claim', value.get('certificate')) for value in resolutions.values()]
|
||||
return response
|
||||
|
||||
async def address_is_mine(self, unknown_address, account):
|
||||
match = await self.ledger.db.get_address(address=unknown_address, account=account)
|
||||
if match is not None:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def get_transaction(self, txid):
|
||||
tx = await self.db.get_transaction(txid=txid)
|
||||
if not tx:
|
||||
@@ -281,17 +369,16 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'is_tip': not is_my_inputs,
|
||||
'nout': txo.position
|
||||
})
|
||||
if is_my_inputs:
|
||||
for txo in tx.other_support_outputs:
|
||||
item['support_info'].append({
|
||||
'address': txo.get_address(account.ledger),
|
||||
'balance_delta': dewies_to_lbc(-txo.amount),
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'is_tip': is_my_inputs,
|
||||
'nout': txo.position
|
||||
})
|
||||
for txo in tx.other_support_outputs:
|
||||
item['support_info'].append({
|
||||
'address': txo.get_address(account.ledger),
|
||||
'balance_delta': dewies_to_lbc(-txo.amount),
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'is_tip': is_my_inputs,
|
||||
'nout': txo.position
|
||||
})
|
||||
for txo in tx.my_abandon_outputs:
|
||||
item['abandon_info'].append({
|
||||
'address': txo.get_address(account.ledger),
|
||||
@@ -304,6 +391,127 @@ class LbryWalletManager(BaseWalletManager):
|
||||
history.append(item)
|
||||
return history
|
||||
|
||||
@staticmethod
|
||||
def get_utxos(account: BaseAccount):
|
||||
return account.get_utxos()
|
||||
|
||||
async def claim_name(self, account, name, amount, claim_dict, certificate=None, claim_address=None):
|
||||
claim = ClaimDict.load_dict(claim_dict)
|
||||
if not claim_address:
|
||||
claim_address = await account.receiving.get_or_create_usable_address()
|
||||
if certificate:
|
||||
claim = claim.sign(
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1, name=name,
|
||||
force_detached=False # TODO: delete it and make True default everywhere when its out
|
||||
)
|
||||
existing_claims = await account.get_claims(
|
||||
claim_name_type__any={'is_claim': 1, 'is_update': 1}, # exclude is_supports
|
||||
claim_name=name
|
||||
)
|
||||
if len(existing_claims) == 0:
|
||||
tx = await Transaction.claim(
|
||||
name, claim, amount, claim_address, [account], account
|
||||
)
|
||||
elif len(existing_claims) == 1:
|
||||
tx = await Transaction.update(
|
||||
existing_claims[0], claim, amount, claim_address, [account], account
|
||||
)
|
||||
else:
|
||||
raise NameError(f"More than one other claim exists with the name '{name}'.")
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], claim_address, claim_dict, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
# TODO: release reserved tx outputs in case anything fails by this point
|
||||
return tx
|
||||
|
||||
async def support_claim(self, claim_name, claim_id, amount, account):
|
||||
holding_address = await account.receiving.get_or_create_usable_address()
|
||||
tx = await Transaction.support(claim_name, claim_id, amount, holding_address, [account], account)
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.old_db.save_supports(claim_id, [{
|
||||
'txid': tx.id,
|
||||
'nout': tx.position,
|
||||
'address': holding_address,
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def tip_claim(self, amount, claim_id, account):
|
||||
claim_to_tip = await self.get_claim_by_claim_id(claim_id)
|
||||
tx = await Transaction.support(
|
||||
claim_to_tip['name'], claim_id, amount, claim_to_tip['address'], [account], account
|
||||
)
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.old_db.save_supports(claim_id, [{
|
||||
'txid': tx.id,
|
||||
'nout': tx.position,
|
||||
'address': claim_to_tip['address'],
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def abandon_claim(self, claim_id, txid, nout, account):
|
||||
claim = await account.get_claim(claim_id=claim_id, txid=txid, nout=nout)
|
||||
if not claim:
|
||||
raise Exception('No claim found for the specified claim_id or txid:nout')
|
||||
|
||||
tx = await Transaction.abandon(claim, [account], account)
|
||||
await account.ledger.broadcast(tx)
|
||||
# TODO: release reserved tx outputs in case anything fails by this point
|
||||
return tx
|
||||
|
||||
async def claim_new_channel(self, channel_name, amount, account):
|
||||
address = await account.receiving.get_or_create_usable_address()
|
||||
cert, key = generate_certificate()
|
||||
tx = await Transaction.claim(channel_name, cert, amount, address, [account], account)
|
||||
await account.ledger.broadcast(tx)
|
||||
account.add_certificate_private_key(tx.outputs[0].ref, key.decode())
|
||||
# TODO: release reserved tx outputs in case anything fails by this point
|
||||
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], address, cert, channel_name, dewies_to_lbc(amount)
|
||||
)])
|
||||
return tx
|
||||
|
||||
def _old_get_temp_claim_info(self, tx, txo, address, claim_dict, name, bid):
|
||||
return {
|
||||
"claim_id": txo.claim_id,
|
||||
"name": name,
|
||||
"amount": bid,
|
||||
"address": address,
|
||||
"txid": tx.id,
|
||||
"nout": txo.position,
|
||||
"value": claim_dict,
|
||||
"height": -1,
|
||||
"claim_sequence": -1,
|
||||
}
|
||||
|
||||
def get_certificates(self, private_key_accounts, exclude_without_key=True, **constraints):
|
||||
return self.db.get_certificates(
|
||||
private_key_accounts=private_key_accounts,
|
||||
exclude_without_key=exclude_without_key,
|
||||
**constraints
|
||||
)
|
||||
|
||||
def update_peer_address(self, peer, address):
|
||||
pass # TODO: Data payments is disabled
|
||||
|
||||
def get_unused_address_for_peer(self, peer):
|
||||
# TODO: Data payments is disabled
|
||||
return self.get_unused_address()
|
||||
|
||||
def add_expected_payment(self, peer, amount):
|
||||
pass # TODO: Data payments is disabled
|
||||
|
||||
def send_points(self, reserved_points, amount):
|
||||
pass # TODO: Data payments is disabled
|
||||
|
||||
def cancel_point_reservation(self, reserved_points):
|
||||
pass # fixme: disabled for now.
|
||||
|
||||
def save(self):
|
||||
for wallet in self.wallets:
|
||||
wallet.save()
|
||||
File renamed without changes.
@@ -0,0 +1,491 @@
|
||||
import logging
|
||||
|
||||
from ecdsa import BadSignatureError
|
||||
from binascii import unhexlify, hexlify
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
|
||||
from lbrynet.schema.address import is_address
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Resolver:
|
||||
|
||||
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network):
|
||||
self.claim_trie_root = claim_trie_root
|
||||
self.height = height
|
||||
self.transaction_class = transaction_class
|
||||
self.hash160_to_address = hash160_to_address
|
||||
self.network = network
|
||||
|
||||
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
|
||||
results = {}
|
||||
for uri in requested_uris:
|
||||
resolution = (resolutions or {}).get(uri, {})
|
||||
if resolution:
|
||||
try:
|
||||
results[uri] = _handle_claim_result(
|
||||
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
|
||||
)
|
||||
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
|
||||
results[uri] = {'error': str(err)}
|
||||
else:
|
||||
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
|
||||
return results
|
||||
|
||||
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
|
||||
result = {}
|
||||
claim_trie_root = self.claim_trie_root
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
certificate = None
|
||||
# parse an included certificate
|
||||
if 'certificate' in resolution:
|
||||
certificate_response = resolution['certificate']['result']
|
||||
certificate_resolution_type = resolution['certificate']['resolution_type']
|
||||
if certificate_resolution_type == "winning" and certificate_response:
|
||||
if 'height' in certificate_response:
|
||||
height = certificate_response['height']
|
||||
depth = self.height - height
|
||||
certificate_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
certificate_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "claim_id":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "sequence":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", certificate_resolution_type)
|
||||
|
||||
if 'certificate' in result:
|
||||
certificate = result['certificate']
|
||||
if 'unverified_claims_in_channel' in resolution:
|
||||
max_results = len(resolution['unverified_claims_in_channel'])
|
||||
result['claims_in_channel'] = max_results
|
||||
else:
|
||||
result['claims_in_channel'] = 0
|
||||
else:
|
||||
result['error'] = "claim not found"
|
||||
result['success'] = False
|
||||
result['uri'] = str(parsed_uri)
|
||||
|
||||
else:
|
||||
certificate = None
|
||||
|
||||
# if this was a resolution for a name, parse the result
|
||||
if 'claim' in resolution:
|
||||
claim_response = resolution['claim']['result']
|
||||
claim_resolution_type = resolution['claim']['resolution_type']
|
||||
if claim_resolution_type == "winning" and claim_response:
|
||||
if 'height' in claim_response:
|
||||
height = claim_response['height']
|
||||
depth = self.height - height
|
||||
claim_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
claim_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "claim_id":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "sequence":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", claim_resolution_type)
|
||||
|
||||
# if this was a resolution for a name in a channel make sure there is only one valid
|
||||
# match
|
||||
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
|
||||
unverified_claims_for_name = resolution['unverified_claims_for_name']
|
||||
|
||||
channel_info = await self.get_channel_claims_page(unverified_claims_for_name,
|
||||
result['certificate'], page=1)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if not claims_in_channel:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
elif len(claims_in_channel) > 1:
|
||||
log.warning("Multiple signed claims for the same name.")
|
||||
winner = pick_winner_from_channel_path_collision(claims_in_channel)
|
||||
if winner:
|
||||
result['claim'] = winner
|
||||
else:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
else:
|
||||
result['claim'] = claims_in_channel[0]
|
||||
|
||||
# parse and validate claims in a channel iteratively into pages of results
|
||||
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
|
||||
ids_to_check = resolution['unverified_claims_in_channel']
|
||||
channel_info = await self.get_channel_claims_page(ids_to_check, result['certificate'],
|
||||
page=page, page_size=page_size)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if claims_in_channel:
|
||||
result['claims_in_channel'] = claims_in_channel
|
||||
elif 'error' not in result:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
|
||||
# invalid signatures can only return outside a channel
|
||||
if result.get('claim', {}).get('has_signature', False):
|
||||
if parsed_uri.path and not result['claim']['signature_is_valid']:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
return result
|
||||
|
||||
async def get_certificate_and_validate_result(self, claim_result):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
|
||||
claim_result['decoded_claim'] = False
|
||||
decoded = None
|
||||
|
||||
if not raw:
|
||||
claim_value = claim_result['value']
|
||||
try:
|
||||
decoded = smart_decode(claim_value)
|
||||
claim_result['value'] = decoded.claim_dict
|
||||
claim_result['decoded_claim'] = True
|
||||
except DecodeError:
|
||||
pass
|
||||
|
||||
if decoded:
|
||||
claim_result['has_signature'] = False
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
|
||||
if 'height' in claim_result and claim_result['height'] is None:
|
||||
claim_result['height'] = -1
|
||||
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id if decoded else None)
|
||||
|
||||
return claim_result
|
||||
|
||||
@staticmethod
|
||||
def prepare_claim_queries(start_position, query_size, channel_claim_infos):
|
||||
queries = [tuple()]
|
||||
names = {}
|
||||
# a table of index counts for the sorted claim ids, including ignored claims
|
||||
absolute_position_index = {}
|
||||
|
||||
block_sorted_infos = sorted(channel_claim_infos.items(), key=lambda x: int(x[1][1]))
|
||||
per_block_infos = {}
|
||||
for claim_id, (name, height) in block_sorted_infos:
|
||||
claims = per_block_infos.get(height, [])
|
||||
claims.append((claim_id, name))
|
||||
per_block_infos[height] = sorted(claims, key=lambda x: int(x[0], 16))
|
||||
|
||||
abs_position = 0
|
||||
|
||||
for height in sorted(per_block_infos.keys(), reverse=True):
|
||||
for claim_id, name in per_block_infos[height]:
|
||||
names[claim_id] = name
|
||||
absolute_position_index[claim_id] = abs_position
|
||||
if abs_position >= start_position:
|
||||
if len(queries[-1]) >= query_size:
|
||||
queries.append(tuple())
|
||||
queries[-1] += (claim_id,)
|
||||
abs_position += 1
|
||||
return queries, names, absolute_position_index
|
||||
|
||||
async def iter_channel_claims_pages(self, queries, claim_positions, claim_names, certificate,
|
||||
page_size=10):
|
||||
# lbryum server returns a dict of {claim_id: (name, claim_height)}
|
||||
# first, sort the claims by block height (and by claim id int value within a block).
|
||||
|
||||
# map the sorted claims into getclaimsbyids queries of query_size claim ids each
|
||||
|
||||
# send the batched queries to lbryum server and iteratively validate and parse
|
||||
# the results, yield a page of results at a time.
|
||||
|
||||
# these results can include those where `signature_is_valid` is False. if they are skipped,
|
||||
# page indexing becomes tricky, as the number of results isn't known until after having
|
||||
# processed them.
|
||||
# TODO: fix ^ in lbrynet.schema
|
||||
|
||||
async def iter_validate_channel_claims():
|
||||
formatted_claims = []
|
||||
for claim_ids in queries:
|
||||
batch_result = await self.network.get_claims_by_ids(*claim_ids)
|
||||
for claim_id in claim_ids:
|
||||
claim = batch_result[claim_id]
|
||||
if claim['name'] == claim_names[claim_id]:
|
||||
formatted_claim = await self.parse_and_validate_claim_result(claim, certificate)
|
||||
formatted_claim['absolute_channel_position'] = claim_positions[
|
||||
claim['claim_id']]
|
||||
formatted_claims.append(formatted_claim)
|
||||
else:
|
||||
log.warning("ignoring claim with name mismatch %s %s", claim['name'],
|
||||
claim['claim_id'])
|
||||
return formatted_claims
|
||||
|
||||
results = []
|
||||
|
||||
for claim in await iter_validate_channel_claims():
|
||||
results.append(claim)
|
||||
|
||||
# if there is a full page of results, yield it
|
||||
if len(results) and len(results) % page_size == 0:
|
||||
return results[-page_size:]
|
||||
|
||||
return results
|
||||
|
||||
async def get_channel_claims_page(self, channel_claim_infos, certificate, page, page_size=10):
|
||||
page = page or 0
|
||||
page_size = max(page_size, 1)
|
||||
if page_size > 500:
|
||||
raise Exception("page size above maximum allowed")
|
||||
start_position = (page - 1) * page_size
|
||||
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
|
||||
channel_claim_infos)
|
||||
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
|
||||
certificate, page_size=page_size)
|
||||
upper_bound = len(claim_positions)
|
||||
if not page:
|
||||
return None, upper_bound
|
||||
if start_position > upper_bound:
|
||||
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
|
||||
return page_generator, upper_bound
|
||||
|
||||
|
||||
# Format amount to be decimal encoded string
|
||||
# Format value to be hex encoded string
|
||||
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
|
||||
def format_amount_value(obj):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in ('amount', 'effective_amount'):
|
||||
if not isinstance(obj[k], float):
|
||||
obj[k] = dewies_to_lbc(obj[k])
|
||||
elif k == 'supports' and isinstance(v, list):
|
||||
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in v]
|
||||
elif isinstance(v, (list, dict)):
|
||||
obj[k] = format_amount_value(v)
|
||||
elif isinstance(obj, list):
|
||||
obj = [format_amount_value(o) for o in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
"""
|
||||
Verify proof for name claim
|
||||
"""
|
||||
|
||||
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
|
||||
claim_sequence, claim_address, supports):
|
||||
r = {
|
||||
'name': name,
|
||||
'value': hexlify(value),
|
||||
'claim_id': claim_id,
|
||||
'txid': txid,
|
||||
'nout': n,
|
||||
'amount': amount,
|
||||
'effective_amount': effective_amount,
|
||||
'height': height,
|
||||
'depth': depth,
|
||||
'claim_sequence': claim_sequence,
|
||||
'address': claim_address,
|
||||
'supports': supports
|
||||
}
|
||||
return r
|
||||
|
||||
def _parse_proof_result(name, result):
|
||||
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
|
||||
supports = result['supports']
|
||||
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
|
||||
if 'transaction' in result:
|
||||
tx = transaction_class(raw=unhexlify(result['transaction']))
|
||||
nOut = result['proof']['nOut']
|
||||
if result['proof']['txhash'] == tx.id:
|
||||
if 0 <= nOut < len(tx.outputs):
|
||||
claim_output = tx.outputs[nOut]
|
||||
effective_amount = claim_output.amount + support_amount
|
||||
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
|
||||
claim_id = result['claim_id']
|
||||
claim_sequence = result['claim_sequence']
|
||||
claim_script = claim_output.script
|
||||
decoded_name = claim_script.values['claim_name'].decode()
|
||||
decoded_value = claim_script.values['claim']
|
||||
if decoded_name == name:
|
||||
return _build_response(name, decoded_value, claim_id,
|
||||
tx.id, nOut, claim_output.amount,
|
||||
effective_amount, claim_sequence,
|
||||
claim_address, supports)
|
||||
return {'error': 'name in proof did not match requested name'}
|
||||
outputs = len(tx['outputs'])
|
||||
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
|
||||
return {'error': "computed txid did not match given transaction: %s vs %s" %
|
||||
(tx.id, result['proof']['txhash'])
|
||||
}
|
||||
return {'error': "didn't receive a transaction with the proof"}
|
||||
return {'error': 'name is not claimed'}
|
||||
|
||||
if 'proof' in result:
|
||||
try:
|
||||
verify_proof(result['proof'], claim_trie_root, name)
|
||||
except InvalidProofError:
|
||||
return {'error': "Proof was invalid"}
|
||||
return _parse_proof_result(name, result)
|
||||
else:
|
||||
return {'error': "proof not in result"}
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
log.warning('Got an invalid claim while parsing certificates, please report: %s', certificate_claim)
|
||||
return False, None
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
return False
|
||||
except Exception as err:
|
||||
log.error("Signature for %s is invalid, reason: %s - %s", claim_address,
|
||||
str(type(err)), err)
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
|
||||
def _decode_claim_result(claim):
|
||||
if 'has_signature' in claim and claim['has_signature']:
|
||||
if not claim['signature_is_valid']:
|
||||
log.warning("lbry://%s#%s has an invalid signature",
|
||||
claim['name'], claim['claim_id'])
|
||||
if 'value' not in claim:
|
||||
log.warning('Got an invalid claim while parsing, please report: %s', claim)
|
||||
claim['hex'] = None
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to parse: missing value"
|
||||
return claim
|
||||
try:
|
||||
decoded = smart_decode(claim['value'])
|
||||
claim_dict = decoded.claim_dict
|
||||
claim['value'] = claim_dict
|
||||
claim['hex'] = hexlify(decoded.serialized)
|
||||
except DecodeError:
|
||||
claim['hex'] = claim['value']
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to decode value"
|
||||
return claim
|
||||
|
||||
|
||||
def _handle_claim_result(results):
|
||||
if not results:
|
||||
#TODO: cannot determine what name we searched for here
|
||||
# we should fix lbryum commands that return None
|
||||
raise UnknownNameError("")
|
||||
|
||||
if 'error' in results:
|
||||
if results['error'] in ['name is not claimed', 'claim not found']:
|
||||
if 'claim_id' in results:
|
||||
raise UnknownClaimID(results['claim_id'])
|
||||
elif 'name' in results:
|
||||
raise UnknownNameError(results['name'])
|
||||
elif 'uri' in results:
|
||||
raise UnknownURI(results['uri'])
|
||||
elif 'outpoint' in results:
|
||||
raise UnknownOutpoint(results['outpoint'])
|
||||
raise Exception(results['error'])
|
||||
|
||||
# case where return value is {'certificate':{'txid', 'value',...},...}
|
||||
if 'certificate' in results:
|
||||
results['certificate'] = _decode_claim_result(results['certificate'])
|
||||
|
||||
# case where return value is {'claim':{'txid','value',...},...}
|
||||
if 'claim' in results:
|
||||
results['claim'] = _decode_claim_result(results['claim'])
|
||||
|
||||
# case where return value is {'txid','value',...}
|
||||
# returned by queries that are not name resolve related
|
||||
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
|
||||
elif 'value' in results:
|
||||
results = _decode_claim_result(results)
|
||||
|
||||
# case where there is no 'certificate', 'value', or 'claim' key
|
||||
elif 'certificate' not in results:
|
||||
msg = f'result in unexpected format:{results}'
|
||||
assert False, msg
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def pick_winner_from_channel_path_collision(claims_in_channel):
|
||||
# we should be doing this by effective amount so we pick the controlling claim, however changing the resolved
|
||||
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
|
||||
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
|
||||
# potentially triggering that.
|
||||
winner = None
|
||||
for claim in claims_in_channel:
|
||||
if not claim['signature_is_valid']:
|
||||
continue
|
||||
if winner is None:
|
||||
winner = claim
|
||||
elif claim['claim_sequence'] < winner['claim_sequence']:
|
||||
winner = claim
|
||||
return winner
|
||||
File renamed without changes.
File renamed without changes.
+53
-36
@@ -1,14 +1,15 @@
|
||||
import hashlib
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
|
||||
import msgpack
|
||||
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.block_processor import BlockProcessor
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
from lbrynet.wallet.server.model import ClaimInfo
|
||||
from lbrynet.extras.wallet.server.model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport
|
||||
|
||||
|
||||
class LBRYBlockProcessor(BlockProcessor):
|
||||
@@ -36,31 +37,32 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
undo_info = []
|
||||
add_undo = undo_info.append
|
||||
update_inputs = set()
|
||||
for etx, txid in txs:
|
||||
for tx, txid in txs:
|
||||
update_inputs.clear()
|
||||
tx = Transaction(etx.serialize())
|
||||
for index, output in enumerate(tx.outputs):
|
||||
if not output.is_claim:
|
||||
continue
|
||||
if output.script.is_claim_name:
|
||||
add_undo(self.advance_claim_name_transaction(output, height, txid, index))
|
||||
elif output.script.is_update_claim:
|
||||
update_input = self.db.get_update_input(output.claim_hash, tx.inputs)
|
||||
if update_input:
|
||||
update_inputs.add(update_input)
|
||||
add_undo(self.advance_update_claim(output, height, txid, index))
|
||||
else:
|
||||
info = (hash_to_hex_str(txid), output.claim_id,)
|
||||
self.logger.error("REJECTED: {} updating {}".format(*info))
|
||||
if tx.has_claims:
|
||||
for index, output in enumerate(tx.outputs):
|
||||
claim = output.claim
|
||||
if isinstance(claim, NameClaim):
|
||||
add_undo(self.advance_claim_name_transaction(output, height, txid, index))
|
||||
elif isinstance(claim, ClaimUpdate):
|
||||
update_input = self.db.get_update_input(claim, tx.inputs)
|
||||
if update_input:
|
||||
update_inputs.add(update_input)
|
||||
add_undo(self.advance_update_claim(output, height, txid, index))
|
||||
else:
|
||||
info = (hash_to_hex_str(txid), hash_to_hex_str(claim.claim_id),)
|
||||
self.logger.error("REJECTED: {} updating {}".format(*info))
|
||||
elif isinstance(claim, ClaimSupport):
|
||||
self.advance_support(claim, txid, index, height, output.value)
|
||||
for txin in tx.inputs:
|
||||
if txin not in update_inputs:
|
||||
abandoned_claim_id = self.db.abandon_spent(txin.txo_ref.tx_ref.hash, txin.txo_ref.position)
|
||||
abandoned_claim_id = self.db.abandon_spent(txin.prev_hash, txin.prev_idx)
|
||||
if abandoned_claim_id:
|
||||
add_undo((abandoned_claim_id, self.db.get_claim_info(abandoned_claim_id)))
|
||||
return undo_info
|
||||
|
||||
def advance_update_claim(self, output: Output, height, txid, nout):
|
||||
claim_id = output.claim_hash
|
||||
def advance_update_claim(self, output, height, txid, nout):
|
||||
claim_id = output.claim.claim_id
|
||||
claim_info = self.claim_info_from_output(output, txid, nout, height)
|
||||
old_claim_info = self.db.get_claim_info(claim_id)
|
||||
self.db.put_claim_id_for_outpoint(old_claim_info.txid, old_claim_info.nout, None)
|
||||
@@ -72,12 +74,13 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
|
||||
return claim_id, old_claim_info
|
||||
|
||||
def advance_claim_name_transaction(self, output: Output, height, txid, nout):
|
||||
claim_id = output.claim_hash
|
||||
def advance_claim_name_transaction(self, output, height, txid, nout):
|
||||
claim_id = claim_id_hash(txid, nout)
|
||||
claim_info = self.claim_info_from_output(output, txid, nout, height)
|
||||
if claim_info.cert_id:
|
||||
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
|
||||
self.db.put_claim_info(claim_id, claim_info)
|
||||
self.db.put_claim_for_name(claim_info.name, claim_id)
|
||||
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
|
||||
return claim_id, None
|
||||
|
||||
@@ -111,8 +114,9 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
if undo_claim_info:
|
||||
self.db.put_claim_info(claim_id, undo_claim_info)
|
||||
if undo_claim_info.cert_id:
|
||||
cert_id = self._checksig(undo_claim_info.value, undo_claim_info.address)
|
||||
cert_id = self._checksig(undo_claim_info.name, undo_claim_info.value, undo_claim_info.address)
|
||||
self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id)
|
||||
self.db.put_claim_for_name(undo_claim_info.name, claim_id)
|
||||
self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id)
|
||||
|
||||
def backup_txs(self, txs):
|
||||
@@ -131,24 +135,37 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.db.batched_flush_claims()
|
||||
return await super().flush(flush_utxos)
|
||||
|
||||
def claim_info_from_output(self, output: Output, txid, nout, height):
|
||||
address = self.coin.address_from_script(output.script.source)
|
||||
name, value, cert_id = output.script.values['claim_name'], output.script.values['claim'], None
|
||||
assert txid and address
|
||||
cert_id = self._checksig(value, address)
|
||||
return ClaimInfo(name, value, txid, nout, output.amount, address, height, cert_id)
|
||||
def advance_support(self, claim_support, txid, nout, height, amount):
|
||||
# TODO: check for more controller claim rules, like takeover or ordering
|
||||
pass
|
||||
|
||||
def _checksig(self, value, address):
|
||||
def claim_info_from_output(self, output, txid, nout, height):
|
||||
amount = output.value
|
||||
address = self.coin.address_from_script(output.pk_script)
|
||||
name, value, cert_id = output.claim.name, output.claim.value, None
|
||||
assert txid and address
|
||||
cert_id = self._checksig(name, value, address)
|
||||
return ClaimInfo(name, value, txid, nout, amount, address, height, cert_id)
|
||||
|
||||
def _checksig(self, name, value, address):
|
||||
try:
|
||||
claim_dict = Claim.from_bytes(value)
|
||||
cert_id = claim_dict.signing_channel_hash
|
||||
parse_lbry_uri(name.decode()) # skip invalid names
|
||||
claim_dict = smart_decode(value)
|
||||
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
|
||||
if not self.should_validate_signatures:
|
||||
return cert_id
|
||||
if cert_id:
|
||||
cert_claim = self.db.get_claim_info(cert_id)
|
||||
if cert_claim:
|
||||
certificate = Claim.from_bytes(cert_claim.value)
|
||||
certificate = smart_decode(cert_claim.value)
|
||||
claim_dict.validate_signature(address, certificate)
|
||||
return cert_id
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
def claim_id_hash(txid, n):
|
||||
# TODO: This should be in lbryschema
|
||||
packed = txid + struct.pack('>I', n)
|
||||
md = hashlib.new('ripemd160')
|
||||
md.update(hashlib.sha256(packed).digest())
|
||||
return md.digest()
|
||||
@@ -1,29 +1,32 @@
|
||||
import struct
|
||||
|
||||
from lbrynet.wallet.script import OutputScript
|
||||
from torba.server.script import ScriptPubKey, OpCodes
|
||||
from torba.server.script import ScriptPubKey, _match_ops, OpCodes
|
||||
from torba.server.util import cachedproperty
|
||||
from torba.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from hashlib import sha256
|
||||
from torba.server.coins import Coin, CoinError
|
||||
|
||||
from lbrynet.extras.wallet.server.opcodes import decode_claim_script, opcodes as lbry_opcodes
|
||||
|
||||
|
||||
class LBC(Coin):
|
||||
from .session import LBRYElectrumX
|
||||
from .block_processor import LBRYBlockProcessor
|
||||
from .tx import LBRYDeserializer
|
||||
from .daemon import LBCDaemon
|
||||
from .db import LBRYDB
|
||||
DAEMON = LBCDaemon
|
||||
SESSIONCLS = LBRYElectrumX
|
||||
BLOCK_PROCESSOR = LBRYBlockProcessor
|
||||
DB = LBRYDB
|
||||
DESERIALIZER = LBRYDeserializer
|
||||
NAME = "LBRY"
|
||||
SHORTNAME = "LBC"
|
||||
NET = "mainnet"
|
||||
BASIC_HEADER_SIZE = 112
|
||||
CHUNK_SIZE = 96
|
||||
XPUB_VERBYTES = bytes.fromhex("0488b21e")
|
||||
XPRV_VERBYTES = bytes.fromhex("0488ade4")
|
||||
XPUB_VERBYTES = bytes.fromhex("019C354f")
|
||||
XPRV_VERBYTES = bytes.fromhex("019C3118")
|
||||
P2PKH_VERBYTE = bytes.fromhex("55")
|
||||
P2SH_VERBYTES = bytes.fromhex("7A")
|
||||
WIF_BYTE = bytes.fromhex("1C")
|
||||
@@ -85,14 +88,28 @@ class LBC(Coin):
|
||||
def claim_address_handler(cls, script):
|
||||
'''Parse a claim script, returns the address
|
||||
'''
|
||||
output = OutputScript(script)
|
||||
if output.is_pay_pubkey_hash:
|
||||
return cls.P2PKH_address_from_hash160(output.values['pubkey_hash'])
|
||||
if output.is_pay_script_hash:
|
||||
return cls.P2SH_address_from_hash160(output.values['script_hash'])
|
||||
if output.is_pay_pubkey:
|
||||
return cls.P2PKH_address_from_pubkey(output.values['pubkey'])
|
||||
if output.is_return_data:
|
||||
decoded = decode_claim_script(script)
|
||||
if not decoded:
|
||||
return None
|
||||
ops = []
|
||||
for op, data, _ in decoded[1]:
|
||||
if not data:
|
||||
ops.append(op)
|
||||
else:
|
||||
ops.append((op, data,))
|
||||
match = _match_ops
|
||||
TO_ADDRESS_OPS = [OpCodes.OP_DUP, OpCodes.OP_HASH160, -1,
|
||||
OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG]
|
||||
TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL]
|
||||
TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG]
|
||||
|
||||
if match(ops, TO_ADDRESS_OPS):
|
||||
return cls.P2PKH_address_from_hash160(ops[2][-1])
|
||||
if match(ops, TO_P2SH_OPS):
|
||||
return cls.P2SH_address_from_hash160(ops[1][-1])
|
||||
if match(ops, TO_PUBKEY_OPS):
|
||||
return cls.P2PKH_address_from_pubkey(ops[0][-1])
|
||||
if ops and ops[0] == OpCodes.OP_RETURN:
|
||||
return None
|
||||
return None
|
||||
|
||||
@@ -104,9 +121,9 @@ class LBC(Coin):
|
||||
if script and script[0] == OpCodes.OP_RETURN:
|
||||
return None
|
||||
if script[0] in [
|
||||
OutputScript.OP_CLAIM_NAME,
|
||||
OutputScript.OP_UPDATE_CLAIM,
|
||||
OutputScript.OP_SUPPORT_CLAIM,
|
||||
lbry_opcodes.OP_CLAIM_NAME,
|
||||
lbry_opcodes.OP_SUPPORT_CLAIM,
|
||||
lbry_opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
return cls.address_to_hashX(cls.claim_address_handler(script))
|
||||
else:
|
||||
File renamed without changes.
@@ -6,16 +6,17 @@ from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.db import DB
|
||||
|
||||
from lbrynet.wallet.server.model import ClaimInfo
|
||||
from lbrynet.extras.wallet.server.model import ClaimInfo
|
||||
|
||||
|
||||
class LBRYDB(DB):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.claim_cache = {}
|
||||
self.claims_for_name_cache = {}
|
||||
self.claims_signed_by_cert_cache = {}
|
||||
self.outpoint_to_claim_id_cache = {}
|
||||
self.claims_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
|
||||
self.claims_db = self.names_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
|
||||
# stores deletes not yet flushed to disk
|
||||
self.pending_abandons = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -23,6 +24,7 @@ class LBRYDB(DB):
|
||||
def close(self):
|
||||
self.batched_flush_claims()
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
self.signatures_db.close()
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
@@ -40,10 +42,12 @@ class LBRYDB(DB):
|
||||
return
|
||||
log_reason('closing claim DBs to re-open', for_sync)
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
self.signatures_db.close()
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.claims_db = self.db_class('claims', for_sync)
|
||||
self.names_db = self.db_class('names', for_sync)
|
||||
self.signatures_db = self.db_class('signatures', for_sync)
|
||||
self.outpoint_to_claim_id_db = self.db_class('outpoint_claim_id', for_sync)
|
||||
self.claim_undo_db = self.db_class('claim_undo', for_sync)
|
||||
@@ -56,18 +60,21 @@ class LBRYDB(DB):
|
||||
|
||||
def batched_flush_claims(self):
|
||||
with self.claims_db.write_batch() as claims_batch:
|
||||
with self.signatures_db.write_batch() as signed_claims_batch:
|
||||
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
|
||||
self.flush_claims(claims_batch, signed_claims_batch, outpoint_batch)
|
||||
with self.names_db.write_batch() as names_batch:
|
||||
with self.signatures_db.write_batch() as signed_claims_batch:
|
||||
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
|
||||
self.flush_claims(claims_batch, names_batch, signed_claims_batch,
|
||||
outpoint_batch)
|
||||
|
||||
def flush_claims(self, batch, signed_claims_batch, outpoint_batch):
|
||||
def flush_claims(self, batch, names_batch, signed_claims_batch, outpoint_batch):
|
||||
flush_start = time.time()
|
||||
write_claim, write_cert = batch.put, signed_claims_batch.put
|
||||
write_claim, write_name, write_cert = batch.put, names_batch.put, signed_claims_batch.put
|
||||
write_outpoint = outpoint_batch.put
|
||||
delete_claim, delete_outpoint = batch.delete, outpoint_batch.delete
|
||||
delete_claim, delete_outpoint, delete_name = batch.delete, outpoint_batch.delete, names_batch.delete
|
||||
delete_cert = signed_claims_batch.delete
|
||||
for claim_id, outpoints in self.pending_abandons.items():
|
||||
claim = self.get_claim_info(claim_id)
|
||||
self.remove_claim_for_name(claim.name, claim_id)
|
||||
if claim.cert_id:
|
||||
self.remove_claim_from_certificate_claims(claim.cert_id, claim_id)
|
||||
self.remove_certificate(claim_id)
|
||||
@@ -79,6 +86,11 @@ class LBRYDB(DB):
|
||||
write_claim(key, claim)
|
||||
else:
|
||||
delete_claim(key)
|
||||
for name, claims in self.claims_for_name_cache.items():
|
||||
if not claims:
|
||||
delete_name(name)
|
||||
else:
|
||||
write_name(name, msgpack.dumps(claims))
|
||||
for cert_id, claims in self.claims_signed_by_cert_cache.items():
|
||||
if not claims:
|
||||
delete_cert(cert_id)
|
||||
@@ -89,13 +101,15 @@ class LBRYDB(DB):
|
||||
write_outpoint(key, claim_id)
|
||||
else:
|
||||
delete_outpoint(key)
|
||||
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints '
|
||||
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints, {:,d} names '
|
||||
'and {:,d} certificates added while {:,d} were abandoned in {:.1f}s, committing...'
|
||||
.format(self.db_height,
|
||||
len(self.claim_cache), len(self.outpoint_to_claim_id_cache),
|
||||
len(self.claims_for_name_cache),
|
||||
len(self.claims_signed_by_cert_cache), len(self.pending_abandons),
|
||||
time.time() - flush_start))
|
||||
self.claim_cache = {}
|
||||
self.claims_for_name_cache = {}
|
||||
self.claims_signed_by_cert_cache = {}
|
||||
self.outpoint_to_claim_id_cache = {}
|
||||
self.pending_abandons = {}
|
||||
@@ -103,6 +117,7 @@ class LBRYDB(DB):
|
||||
def assert_flushed(self, flush_data):
|
||||
super().assert_flushed(flush_data)
|
||||
assert not self.claim_cache
|
||||
assert not self.claims_for_name_cache
|
||||
assert not self.claims_signed_by_cert_cache
|
||||
assert not self.outpoint_to_claim_id_cache
|
||||
assert not self.pending_abandons
|
||||
@@ -127,6 +142,27 @@ class LBRYDB(DB):
|
||||
key = tx_hash + struct.pack('>I', tx_idx)
|
||||
return self.outpoint_to_claim_id_cache.get(key) or self.outpoint_to_claim_id_db.get(key)
|
||||
|
||||
def get_claims_for_name(self, name):
|
||||
if name in self.claims_for_name_cache:
|
||||
return self.claims_for_name_cache[name]
|
||||
db_claims = self.names_db.get(name)
|
||||
return msgpack.loads(db_claims) if db_claims else {}
|
||||
|
||||
def put_claim_for_name(self, name, claim_id):
|
||||
self.logger.info("[+] Adding claim {} for name {}.".format(hash_to_hex_str(claim_id), name))
|
||||
claims = self.get_claims_for_name(name)
|
||||
claims.setdefault(claim_id, max(claims.values() or [0]) + 1)
|
||||
self.claims_for_name_cache[name] = claims
|
||||
|
||||
def remove_claim_for_name(self, name, claim_id):
|
||||
self.logger.info("[-] Removing claim from name: {} - {}".format(hash_to_hex_str(claim_id), name))
|
||||
claims = self.get_claims_for_name(name)
|
||||
claim_n = claims.pop(claim_id)
|
||||
for _claim_id, number in claims.items():
|
||||
if number > claim_n:
|
||||
claims[_claim_id] = number - 1
|
||||
self.claims_for_name_cache[name] = claims
|
||||
|
||||
def get_signed_claim_ids_by_cert_id(self, cert_id):
|
||||
if cert_id in self.claims_signed_by_cert_cache:
|
||||
return self.claims_signed_by_cert_cache[cert_id]
|
||||
@@ -134,20 +170,17 @@ class LBRYDB(DB):
|
||||
return msgpack.loads(db_claims, use_list=True) if db_claims else []
|
||||
|
||||
def put_claim_id_signed_by_cert_id(self, cert_id, claim_id):
|
||||
msg = "[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
self.logger.info("[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
|
||||
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
|
||||
certs.append(claim_id)
|
||||
self.claims_signed_by_cert_cache[cert_id] = certs
|
||||
|
||||
def remove_certificate(self, cert_id):
|
||||
msg = "[-] Removing certificate: {}".format(hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
self.logger.info("[-] Removing certificate: {}".format(hash_to_hex_str(cert_id)))
|
||||
self.claims_signed_by_cert_cache[cert_id] = []
|
||||
|
||||
def remove_claim_from_certificate_claims(self, cert_id, claim_id):
|
||||
msg = "[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
self.logger.info("[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
|
||||
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
|
||||
if claim_id in certs:
|
||||
certs.remove(claim_id)
|
||||
@@ -161,12 +194,13 @@ class LBRYDB(DB):
|
||||
self.logger.info("[+] Adding claim info for: {}".format(hash_to_hex_str(claim_id)))
|
||||
self.claim_cache[claim_id] = claim_info.serialized
|
||||
|
||||
def get_update_input(self, claim_id, inputs):
|
||||
def get_update_input(self, claim, inputs):
|
||||
claim_id = claim.claim_id
|
||||
claim_info = self.get_claim_info(claim_id)
|
||||
if not claim_info:
|
||||
return False
|
||||
for input in inputs:
|
||||
if (input.txo_ref.tx_ref.hash, input.txo_ref.position) == (claim_info.txid, claim_info.nout):
|
||||
if input.prev_hash == claim_info.txid and input.prev_idx == claim_info.nout:
|
||||
return input
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
from collections import namedtuple
|
||||
import msgpack
|
||||
from torba.server.util import cachedproperty
|
||||
# Classes representing data and their serializers, if any.
|
||||
|
||||
|
||||
class ClaimInfo(namedtuple("NameClaim", "name value txid nout amount address height cert_id")):
|
||||
'''Claim information as its stored on database'''
|
||||
|
||||
@classmethod
|
||||
def from_serialized(cls, serialized):
|
||||
return cls(*msgpack.loads(serialized))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
return msgpack.dumps(self)
|
||||
|
||||
|
||||
class NameClaim(namedtuple("NameClaim", "name value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimUpdate(namedtuple("ClaimUpdate", "name claim_id value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimSupport(namedtuple("ClaimSupport", "name claim_id")):
|
||||
pass
|
||||
|
||||
|
||||
class LBRYTx(namedtuple("Tx", "version inputs outputs locktime")):
|
||||
'''Transaction that can contain claim, update or support in its outputs.'''
|
||||
|
||||
@cachedproperty
|
||||
def is_coinbase(self):
|
||||
return self.inputs[0].is_coinbase
|
||||
|
||||
@cachedproperty
|
||||
def has_claims(self):
|
||||
for output in self.outputs:
|
||||
if output.claim:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class TxClaimOutput(namedtuple("TxClaimOutput", "value pk_script claim")):
|
||||
pass
|
||||
@@ -0,0 +1,126 @@
|
||||
import struct
|
||||
from torba.server.enum import Enumeration
|
||||
from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate
|
||||
# TODO: Take this to lbryschema (it's also on lbryum and lbryum-server)
|
||||
|
||||
|
||||
opcodes = Enumeration("Opcodes", [
|
||||
("OP_0", 0), ("OP_PUSHDATA1", 76), "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", "OP_RESERVED",
|
||||
"OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7",
|
||||
"OP_8", "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
|
||||
"OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", "OP_ELSE", "OP_ENDIF",
|
||||
"OP_VERIFY",
|
||||
"OP_RETURN", "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", "OP_2OVER",
|
||||
"OP_2ROT", "OP_2SWAP",
|
||||
"OP_IFDUP", "OP_DEPTH", "OP_DROP", "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL",
|
||||
"OP_ROT",
|
||||
"OP_SWAP", "OP_TUCK", "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", "OP_INVERT",
|
||||
"OP_AND",
|
||||
"OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", "OP_RESERVED1", "OP_RESERVED2", "OP_1ADD",
|
||||
"OP_1SUB", "OP_2MUL",
|
||||
"OP_2DIV", "OP_NEGATE", "OP_ABS", "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL",
|
||||
"OP_DIV",
|
||||
"OP_MOD", "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR",
|
||||
"OP_NUMEQUAL", "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN",
|
||||
"OP_GREATERTHAN", "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
|
||||
"OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160",
|
||||
"OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
|
||||
"OP_CHECKMULTISIGVERIFY", "OP_NOP1", "OP_NOP2", "OP_NOP3", "OP_NOP4", "OP_NOP5",
|
||||
"OP_CLAIM_NAME",
|
||||
"OP_SUPPORT_CLAIM", "OP_UPDATE_CLAIM",
|
||||
("OP_SINGLEBYTE_END", 0xF0),
|
||||
("OP_DOUBLEBYTE_BEGIN", 0xF000),
|
||||
"OP_PUBKEY", "OP_PUBKEYHASH",
|
||||
("OP_INVALIDOPCODE", 0xFFFF),
|
||||
])
|
||||
|
||||
|
||||
def script_GetOp(bytes):
|
||||
i = 0
|
||||
while i < len(bytes):
|
||||
vch = None
|
||||
opcode = bytes[i]
|
||||
i += 1
|
||||
if opcode <= opcodes.OP_PUSHDATA4:
|
||||
nSize = opcode
|
||||
if opcode == opcodes.OP_PUSHDATA1:
|
||||
nSize = bytes[i]
|
||||
i += 1
|
||||
elif opcode == opcodes.OP_PUSHDATA2:
|
||||
(nSize,) = struct.unpack_from('<H', bytes, i)
|
||||
i += 2
|
||||
elif opcode == opcodes.OP_PUSHDATA4:
|
||||
(nSize,) = struct.unpack_from('<I', bytes, i)
|
||||
i += 4
|
||||
if i + nSize > len(bytes):
|
||||
vch = "_INVALID_" + bytes[i:]
|
||||
i = len(bytes)
|
||||
else:
|
||||
vch = bytes[i:i + nSize]
|
||||
i += nSize
|
||||
yield (opcode, vch, i)
|
||||
|
||||
|
||||
def decode_claim_script(bytes_script):
|
||||
try:
|
||||
decoded_script = [x for x in script_GetOp(bytes_script)]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return None
|
||||
if len(decoded_script) <= 6:
|
||||
return False
|
||||
op = 0
|
||||
claim_type = decoded_script[op][0]
|
||||
if claim_type == opcodes.OP_UPDATE_CLAIM:
|
||||
if len(decoded_script) <= 7:
|
||||
return False
|
||||
if claim_type not in [
|
||||
opcodes.OP_CLAIM_NAME,
|
||||
opcodes.OP_SUPPORT_CLAIM,
|
||||
opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
return False
|
||||
op += 1
|
||||
value = None
|
||||
claim_id = None
|
||||
claim = None
|
||||
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
|
||||
return False
|
||||
name = decoded_script[op][1]
|
||||
op += 1
|
||||
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
|
||||
return False
|
||||
if decoded_script[0][0] in [
|
||||
opcodes.OP_SUPPORT_CLAIM,
|
||||
opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
claim_id = decoded_script[op][1]
|
||||
if len(claim_id) != 20:
|
||||
return False
|
||||
else:
|
||||
value = decoded_script[op][1]
|
||||
op += 1
|
||||
if decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
value = decoded_script[op][1]
|
||||
op += 1
|
||||
if decoded_script[op][0] != opcodes.OP_2DROP:
|
||||
return False
|
||||
op += 1
|
||||
if decoded_script[op][0] != opcodes.OP_DROP and decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
|
||||
return False
|
||||
elif decoded_script[op][0] != opcodes.OP_2DROP and decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
return False
|
||||
op += 1
|
||||
if decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
|
||||
if name is None or value is None:
|
||||
return False
|
||||
claim = NameClaim(name, value)
|
||||
elif decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
if name is None or value is None or claim_id is None:
|
||||
return False
|
||||
claim = ClaimUpdate(name, claim_id, value)
|
||||
elif decoded_script[0][0] == opcodes.OP_SUPPORT_CLAIM:
|
||||
if name is None or claim_id is None:
|
||||
return False
|
||||
claim = ClaimSupport(name, claim_id)
|
||||
return claim, decoded_script[op:]
|
||||
@@ -1,5 +1,4 @@
|
||||
import math
|
||||
import unicodedata as uda
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
from torba.rpc.jsonrpc import RPCError
|
||||
@@ -7,9 +6,10 @@ from torba.server.hash import hash_to_hex_str
|
||||
from torba.server.session import ElectrumX
|
||||
from torba.server import util
|
||||
|
||||
from lbrynet.schema.uri import parse_lbry_uri, CLAIM_ID_MAX_LENGTH, URIParseError
|
||||
from lbrynet.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbrynet.wallet.server.db import LBRYDB
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.error import URIParseError
|
||||
from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbrynet.extras.wallet.server.db import LBRYDB
|
||||
|
||||
|
||||
class LBRYElectrumX(ElectrumX):
|
||||
@@ -131,29 +131,22 @@ class LBRYElectrumX(ElectrumX):
|
||||
claim_ids = self.get_claim_ids_signed_by(certificate_id)
|
||||
return await self.batched_formatted_claims_from_daemon(claim_ids)
|
||||
|
||||
def claimtrie_getclaimssignedbyidminimal(self, certificate_id):
|
||||
claim_ids = self.get_claim_ids_signed_by(certificate_id)
|
||||
ret = []
|
||||
for claim_id in claim_ids:
|
||||
raw_claim_id = unhexlify(claim_id)[::-1]
|
||||
info = self.db.get_claim_info(raw_claim_id)
|
||||
if info:
|
||||
ret.append({
|
||||
'claim_id': claim_id,
|
||||
'height': info.height,
|
||||
'name': info.name.decode()
|
||||
})
|
||||
return ret
|
||||
|
||||
def get_claim_ids_signed_by(self, certificate_id):
|
||||
raw_certificate_id = unhexlify(certificate_id)[::-1]
|
||||
raw_claim_ids = self.db.get_signed_claim_ids_by_cert_id(raw_certificate_id)
|
||||
return list(map(hash_to_hex_str, raw_claim_ids))
|
||||
|
||||
def get_signed_claims_with_name_for_channel(self, channel_id, name):
|
||||
claim_ids_for_name = list(self.db.get_claims_for_name(name.encode('ISO-8859-1')).keys())
|
||||
claim_ids_for_name = set(map(hash_to_hex_str, claim_ids_for_name))
|
||||
channel_claim_ids = set(self.get_claim_ids_signed_by(channel_id))
|
||||
return claim_ids_for_name.intersection(channel_claim_ids)
|
||||
|
||||
async def claimtrie_getclaimssignedbynthtoname(self, name, n):
|
||||
claim = self.claimtrie_getnthclaimforname(name, n)
|
||||
if claim and 'claim_id' in claim:
|
||||
return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim['claim_id']))
|
||||
n = int(n)
|
||||
for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
|
||||
if n == sequence:
|
||||
return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim_id))
|
||||
|
||||
async def claimtrie_getclaimsintx(self, txid):
|
||||
# TODO: this needs further discussion.
|
||||
@@ -168,27 +161,28 @@ class LBRYElectrumX(ElectrumX):
|
||||
if proof_has_winning_claim(proof):
|
||||
tx_hash, nout = proof['txhash'], int(proof['nOut'])
|
||||
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
|
||||
result['transaction'] = transaction_info['hex'] # should have never included this (or the call to get it)
|
||||
result['transaction'] = transaction_info['hex']
|
||||
result['height'] = (self.db.db_height - transaction_info['confirmations']) + 1
|
||||
raw_claim_id = self.db.get_claim_id_from_outpoint(unhexlify(tx_hash)[::-1], nout)
|
||||
claim_id = hexlify(raw_claim_id[::-1]).decode()
|
||||
claim = await self.claimtrie_getclaimbyid(claim_id)
|
||||
result.update(claim)
|
||||
|
||||
sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
|
||||
if sequence:
|
||||
claim_id = hexlify(raw_claim_id[::-1]).decode()
|
||||
claim_info = await self.daemon.getclaimbyid(claim_id)
|
||||
if not claim_info or not claim_info.get('value'):
|
||||
claim_info = await self.slow_get_claim_by_id_using_name(claim_id)
|
||||
result['claim_sequence'] = sequence
|
||||
result['claim_id'] = claim_id
|
||||
supports = self.format_supports_from_daemon(claim_info.get('supports', [])) # fixme: lbrycrd#124
|
||||
result['supports'] = supports
|
||||
else:
|
||||
self.logger.warning('tx has no claims in db: %s %s', tx_hash, nout)
|
||||
return result
|
||||
|
||||
async def claimtrie_getnthclaimforname(self, name, n):
|
||||
n = int(n)
|
||||
result = await self.claimtrie_getclaimsforname(name)
|
||||
if 'claims' in result and len(result['claims']) > n >= 0:
|
||||
# TODO: revist this after lbrycrd_#209 to see if we can sort by claim_sequence at this point
|
||||
result['claims'].sort(key=lambda c: (int(c['height']), int(c['nout'])))
|
||||
result['claims'][n]['claim_sequence'] = n
|
||||
return result['claims'][n]
|
||||
|
||||
async def claimtrie_getpartialmatch(self, name, part):
|
||||
result = await self.claimtrie_getclaimsforname(name)
|
||||
if 'claims' in result:
|
||||
return next(filter(lambda x: x['claim_id'].starts_with(part), result['claims']), None)
|
||||
for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
|
||||
if n == sequence:
|
||||
return await self.claimtrie_getclaimbyid(hash_to_hex_str(claim_id))
|
||||
|
||||
async def claimtrie_getclaimsforname(self, name):
|
||||
claims = await self.daemon.getclaimsforname(name)
|
||||
@@ -204,59 +198,51 @@ class LBRYElectrumX(ElectrumX):
|
||||
async def batched_formatted_claims_from_daemon(self, claim_ids):
|
||||
claims = await self.daemon.getclaimsbyids(claim_ids)
|
||||
result = []
|
||||
for claim in claims:
|
||||
for claim, claim_id in zip(claims, claim_ids):
|
||||
if claim and claim.get('value'):
|
||||
result.append(self.format_claim_from_daemon(claim))
|
||||
else:
|
||||
recovered_claim = await self.slow_get_claim_by_id_using_name(claim_id)
|
||||
if recovered_claim:
|
||||
result.append(self.format_claim_from_daemon(recovered_claim))
|
||||
return result
|
||||
|
||||
def format_claim_from_daemon(self, claim, name=None):
|
||||
"""Changes the returned claim data to the format expected by lbrynet and adds missing fields."""
|
||||
|
||||
'''Changes the returned claim data to the format expected by lbrynet and adds missing fields.'''
|
||||
if not claim:
|
||||
return {}
|
||||
|
||||
# this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd
|
||||
# it will be fixed after the lbrycrd upstream merge to v17 is done
|
||||
# it originated as a fear of terminals not supporting unicode. alas, they all do
|
||||
|
||||
if 'name' in claim:
|
||||
name = claim['name'].encode('ISO-8859-1').decode()
|
||||
name = name or claim['name']
|
||||
claim_id = claim['claimId']
|
||||
raw_claim_id = unhexlify(claim_id)[::-1]
|
||||
info = self.db.get_claim_info(raw_claim_id)
|
||||
if not info:
|
||||
# raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
|
||||
if not self.db.get_claim_info(raw_claim_id):
|
||||
#raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
|
||||
return {}
|
||||
address = info.address.decode()
|
||||
supports = self.format_supports_from_daemon(claim.get('supports', []))
|
||||
address = self.db.get_claim_info(raw_claim_id).address.decode()
|
||||
sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
|
||||
if not sequence:
|
||||
return {}
|
||||
supports = self.format_supports_from_daemon(claim.get('supports', [])) # fixme: lbrycrd#124
|
||||
|
||||
amount = get_from_possible_keys(claim, 'amount', 'nAmount')
|
||||
height = get_from_possible_keys(claim, 'height', 'nHeight')
|
||||
effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount')
|
||||
valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight')
|
||||
|
||||
result = {
|
||||
return {
|
||||
"name": name,
|
||||
"claim_id": claim['claimId'],
|
||||
"txid": claim['txid'],
|
||||
"nout": claim['n'],
|
||||
"amount": amount,
|
||||
"depth": self.db.db_height - height + 1,
|
||||
"depth": self.db.db_height - height,
|
||||
"height": height,
|
||||
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
|
||||
"claim_sequence": sequence, # from index
|
||||
"address": address, # from index
|
||||
"supports": supports,
|
||||
"supports": supports, # fixme: to be included in lbrycrd#124
|
||||
"effective_amount": effective_amount,
|
||||
"valid_at_height": valid_at_height
|
||||
"valid_at_height": valid_at_height # TODO PR lbrycrd to include it
|
||||
}
|
||||
if 'claim_sequence' in claim:
|
||||
# TODO: ensure that lbrycrd #209 fills in this value
|
||||
result['claim_sequence'] = claim['claim_sequence']
|
||||
else:
|
||||
result['claim_sequence'] = -1
|
||||
if 'normalized_name' in claim:
|
||||
result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode()
|
||||
return result
|
||||
|
||||
def format_supports_from_daemon(self, supports):
|
||||
return [[support['txid'], support['n'], get_from_possible_keys(support, 'amount', 'nAmount')] for
|
||||
@@ -265,6 +251,8 @@ class LBRYElectrumX(ElectrumX):
|
||||
async def claimtrie_getclaimbyid(self, claim_id):
|
||||
self.assert_claim_id(claim_id)
|
||||
claim = await self.daemon.getclaimbyid(claim_id)
|
||||
if not claim or not claim.get('value'):
|
||||
claim = await self.slow_get_claim_by_id_using_name(claim_id)
|
||||
return self.format_claim_from_daemon(claim)
|
||||
|
||||
async def claimtrie_getclaimsbyids(self, *claim_ids):
|
||||
@@ -291,16 +279,20 @@ class LBRYElectrumX(ElectrumX):
|
||||
pass
|
||||
raise RPCError(1, f'{value} should be a claim id hash')
|
||||
|
||||
def normalize_name(self, name):
|
||||
# this is designed to match lbrycrd; change it here if it changes there
|
||||
return uda.normalize('NFD', name).casefold()
|
||||
|
||||
def claim_matches_name(self, claim, name):
|
||||
if not name:
|
||||
return False
|
||||
if 'normalized_name' in claim:
|
||||
return self.normalize_name(name) == claim['normalized_name']
|
||||
return name == claim['name']
|
||||
async def slow_get_claim_by_id_using_name(self, claim_id):
|
||||
# TODO: temporary workaround for a lbrycrd bug on indexing. Should be removed when it gets stable
|
||||
raw_claim_id = unhexlify(claim_id)[::-1]
|
||||
claim = self.db.get_claim_info(raw_claim_id)
|
||||
if claim:
|
||||
name = claim.name.decode('ISO-8859-1')
|
||||
claims = await self.daemon.getclaimsforname(name)
|
||||
for claim in claims['claims']:
|
||||
if claim['claimId'] == claim_id:
|
||||
claim['name'] = name
|
||||
self.logger.warning(
|
||||
'Recovered a claim missing from lbrycrd index: %s %s', name, claim_id
|
||||
)
|
||||
return claim
|
||||
|
||||
async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None):
|
||||
# TODO: this thing is huge, refactor
|
||||
@@ -320,11 +312,8 @@ class LBRYElectrumX(ElectrumX):
|
||||
|
||||
# TODO: this is also done on the else, refactor
|
||||
if parsed_uri.claim_id:
|
||||
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
|
||||
certificate_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
|
||||
else:
|
||||
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.name):
|
||||
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if certificate_info and certificate_info['name'] == parsed_uri.name:
|
||||
certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
|
||||
elif parsed_uri.claim_sequence:
|
||||
certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
|
||||
@@ -338,28 +327,26 @@ class LBRYElectrumX(ElectrumX):
|
||||
if certificate and 'claim_id' not in certificate['result']:
|
||||
return result
|
||||
|
||||
if certificate:
|
||||
if certificate and not parsed_uri.path:
|
||||
result['certificate'] = certificate
|
||||
channel_id = certificate['result']['claim_id']
|
||||
claims_in_channel = self.claimtrie_getclaimssignedbyidminimal(channel_id)
|
||||
if not parsed_uri.path:
|
||||
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims_in_channel}
|
||||
else:
|
||||
# making an assumption that there aren't case conflicts on an existing channel
|
||||
norm_path = self.normalize_name(parsed_uri.path)
|
||||
result['unverified_claims_for_name'] = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims_in_channel
|
||||
if self.normalize_name(claim['name']) == norm_path}
|
||||
claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id)
|
||||
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims_in_channel if claim}
|
||||
elif certificate:
|
||||
result['certificate'] = certificate
|
||||
channel_id = certificate['result']['claim_id']
|
||||
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path)
|
||||
claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name)
|
||||
|
||||
claims_in_channel = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims}
|
||||
result['unverified_claims_for_name'] = claims_in_channel
|
||||
else:
|
||||
claim = None
|
||||
if parsed_uri.claim_id:
|
||||
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
|
||||
claim_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
|
||||
else:
|
||||
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if claim_info and self.claim_matches_name(claim_info, parsed_uri.name):
|
||||
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if claim_info and claim_info['name'] == parsed_uri.name:
|
||||
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
|
||||
elif parsed_uri.claim_sequence:
|
||||
claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
|
||||
@@ -382,7 +369,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
'result': certificate}
|
||||
result['certificate'] = certificate
|
||||
result['claim'] = claim
|
||||
|
||||
return result
|
||||
|
||||
async def claimtrie_getvalueforuris(self, block_hash, *uris):
|
||||
@@ -0,0 +1,21 @@
|
||||
from torba.server.tx import Deserializer
|
||||
from lbrynet.extras.wallet.server.opcodes import decode_claim_script
|
||||
from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx
|
||||
|
||||
|
||||
class LBRYDeserializer(Deserializer):
|
||||
|
||||
def _read_output(self):
|
||||
value = self._read_le_int64()
|
||||
script = self._read_varbytes() # pk_script
|
||||
claim = decode_claim_script(script)
|
||||
claim = claim[0] if claim else None
|
||||
return TxClaimOutput(value, script, claim)
|
||||
|
||||
def read_tx(self):
|
||||
return LBRYTx(
|
||||
self._read_le_int32(), # version
|
||||
self._read_inputs(), # inputs
|
||||
self._read_outputs(), # outputs
|
||||
self._read_le_uint32() # locktime
|
||||
)
|
||||
@@ -1,21 +1,13 @@
|
||||
import struct
|
||||
import hashlib
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Optional
|
||||
from typing import List, Iterable, Optional
|
||||
|
||||
import ecdsa
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives.serialization import load_der_public_key
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
|
||||
from ecdsa.util import sigencode_der
|
||||
|
||||
from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput, ReadOnlyList
|
||||
from torba.client.hash import hash160, sha256, Base58
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.wallet.account import Account
|
||||
from lbrynet.wallet.script import InputScript, OutputScript
|
||||
from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput
|
||||
from torba.client.hash import hash160
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.extras.wallet.account import Account
|
||||
from lbrynet.extras.wallet.script import InputScript, OutputScript
|
||||
|
||||
|
||||
class Input(BaseInput):
|
||||
@@ -27,11 +19,12 @@ class Output(BaseOutput):
|
||||
script: OutputScript
|
||||
script_class = OutputScript
|
||||
|
||||
__slots__ = 'channel', 'private_key'
|
||||
__slots__ = '_claim_dict', 'channel', 'private_key'
|
||||
|
||||
def __init__(self, *args, channel: Optional['Output'] = None,
|
||||
private_key: Optional[str] = None, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._claim_dict = None
|
||||
self.channel = channel
|
||||
self.private_key = private_key
|
||||
|
||||
@@ -51,17 +44,14 @@ class Output(BaseOutput):
|
||||
return self.script.is_claim_name or self.script.is_update_claim
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
def claim_id(self) -> str:
|
||||
if self.script.is_claim_name:
|
||||
return hash160(self.tx_ref.hash + struct.pack('>I', self.position))
|
||||
claim_id = hash160(self.tx_ref.hash + struct.pack('>I', self.position))
|
||||
elif self.script.is_update_claim or self.script.is_support_claim:
|
||||
return self.script.values['claim_id']
|
||||
claim_id = self.script.values['claim_id']
|
||||
else:
|
||||
raise ValueError('No claim_id associated.')
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
return hexlify(claim_id[::-1]).decode()
|
||||
|
||||
@property
|
||||
def claim_name(self) -> str:
|
||||
@@ -70,16 +60,26 @@ class Output(BaseOutput):
|
||||
raise ValueError('No claim_name associated.')
|
||||
|
||||
@property
|
||||
def claim(self) -> Claim:
|
||||
def claim(self) -> ClaimDict:
|
||||
if self.is_claim:
|
||||
if not isinstance(self.script.values['claim'], Claim):
|
||||
self.script.values['claim'] = Claim.from_bytes(self.script.values['claim'])
|
||||
return self.script.values['claim']
|
||||
return smart_decode(self.script.values['claim'])
|
||||
raise ValueError('Only claim name and claim update have the claim payload.')
|
||||
|
||||
@property
|
||||
def claim_dict(self) -> dict:
|
||||
if self._claim_dict is None:
|
||||
self._claim_dict = self.claim.claim_dict
|
||||
return self._claim_dict
|
||||
|
||||
@property
|
||||
def permanent_url(self) -> str:
|
||||
if self.script.is_claim_involved:
|
||||
if self.channel is not None:
|
||||
return "{}#{}/{}".format(
|
||||
self.channel.claim_name,
|
||||
self.channel.claim_id,
|
||||
self.claim_name
|
||||
)
|
||||
return f"{self.claim_name}#{self.claim_id}"
|
||||
raise ValueError('No claim associated.')
|
||||
|
||||
@@ -87,75 +87,11 @@ class Output(BaseOutput):
|
||||
def has_private_key(self):
|
||||
return self.private_key is not None
|
||||
|
||||
def is_signed_by(self, channel: 'Output', ledger=None):
|
||||
if self.claim.unsigned_payload:
|
||||
pieces = [
|
||||
Base58.decode(self.get_address(ledger)),
|
||||
self.claim.unsigned_payload,
|
||||
self.claim.signing_channel_hash[::-1]
|
||||
]
|
||||
else:
|
||||
pieces = [
|
||||
self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.claim.signing_channel_hash,
|
||||
self.claim.to_message_bytes()
|
||||
]
|
||||
digest = sha256(b''.join(pieces))
|
||||
public_key = load_der_public_key(channel.claim.channel.public_key_bytes, default_backend())
|
||||
hash = hashes.SHA256()
|
||||
signature = hexlify(self.claim.signature)
|
||||
r = int(signature[:int(len(signature)/2)], 16)
|
||||
s = int(signature[int(len(signature)/2):], 16)
|
||||
encoded_sig = sigencode_der(r, s, len(signature)*4)
|
||||
public_key.verify(encoded_sig, digest, ec.ECDSA(Prehashed(hash)))
|
||||
return True
|
||||
|
||||
def sign(self, channel: 'Output', first_input_id=None):
|
||||
self.channel = channel
|
||||
self.claim.signing_channel_hash = channel.claim_hash
|
||||
digest = sha256(b''.join([
|
||||
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.claim.signing_channel_hash,
|
||||
self.claim.to_message_bytes()
|
||||
]))
|
||||
private_key = ecdsa.SigningKey.from_pem(channel.private_key, hashfunc=hashlib.sha256)
|
||||
self.claim.signature = private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.script.generate()
|
||||
|
||||
def clear_signature(self):
|
||||
self.channel = None
|
||||
self.claim.clear_signature()
|
||||
|
||||
def generate_channel_private_key(self):
|
||||
private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
|
||||
self.private_key = private_key.to_pem().decode()
|
||||
self.claim.channel.public_key_bytes = private_key.get_verifying_key().to_der()
|
||||
self.script.generate()
|
||||
return self.private_key
|
||||
|
||||
def is_channel_private_key(self, private_key_pem):
|
||||
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=hashlib.sha256)
|
||||
return self.claim.channel.public_key_bytes == private_key.get_verifying_key().to_der()
|
||||
|
||||
@classmethod
|
||||
def pay_claim_name_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim: Claim, pubkey_hash: bytes) -> 'Output':
|
||||
cls, amount: int, claim_name: str, claim: bytes, pubkey_hash: bytes) -> 'Output':
|
||||
script = cls.script_class.pay_claim_name_pubkey_hash(
|
||||
claim_name.encode(), claim, pubkey_hash)
|
||||
txo = cls(amount, script)
|
||||
return txo
|
||||
|
||||
@classmethod
|
||||
def pay_update_claim_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim_id: str, claim: Claim, pubkey_hash: bytes) -> 'Output':
|
||||
script = cls.script_class.pay_update_claim_pubkey_hash(
|
||||
claim_name.encode(), unhexlify(claim_id)[::-1], claim, pubkey_hash)
|
||||
txo = cls(amount, script)
|
||||
return txo
|
||||
|
||||
@classmethod
|
||||
def pay_support_pubkey_hash(cls, amount: int, claim_name: str, claim_id: str, pubkey_hash: bytes) -> 'Output':
|
||||
script = cls.script_class.pay_support_pubkey_hash(claim_name.encode(), unhexlify(claim_id)[::-1], pubkey_hash)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
@@ -163,15 +99,24 @@ class Output(BaseOutput):
|
||||
script = cls.script_class.purchase_claim_pubkey_hash(unhexlify(claim_id)[::-1], pubkey_hash)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def pay_update_claim_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim_id: str, claim: bytes, pubkey_hash: bytes) -> 'Output':
|
||||
script = cls.script_class.pay_update_claim_pubkey_hash(
|
||||
claim_name.encode(), unhexlify(claim_id)[::-1], claim, pubkey_hash)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def pay_support_pubkey_hash(cls, amount: int, claim_name: str, claim_id: str, pubkey_hash: bytes) -> 'Output':
|
||||
script = cls.script_class.pay_support_pubkey_hash(claim_name.encode(), unhexlify(claim_id)[::-1], pubkey_hash)
|
||||
return cls(amount, script)
|
||||
|
||||
|
||||
class Transaction(BaseTransaction):
|
||||
|
||||
input_class = Input
|
||||
output_class = Output
|
||||
|
||||
outputs: ReadOnlyList[Output]
|
||||
inputs: ReadOnlyList[Input]
|
||||
|
||||
@classmethod
|
||||
def pay(cls, amount: int, address: bytes, funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
@@ -179,52 +124,46 @@ class Transaction(BaseTransaction):
|
||||
return cls.create([], [output], funding_accounts, change_account)
|
||||
|
||||
@classmethod
|
||||
def claim_create(
|
||||
cls, name: str, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
def claim(cls, name: str, meta: ClaimDict, amount: int, holding_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
claim_output = Output.pay_claim_name_pubkey_hash(
|
||||
amount, name, claim, ledger.address_to_hash160(holding_address)
|
||||
amount, name, meta.serialized, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
if signing_channel is not None:
|
||||
claim_output.sign(signing_channel, b'placeholder txid:nout')
|
||||
return cls.create([], [claim_output], funding_accounts, change_account, sign=False)
|
||||
|
||||
@classmethod
|
||||
def claim_update(
|
||||
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
updated_claim = Output.pay_update_claim_pubkey_hash(
|
||||
amount, previous_claim.claim_name, previous_claim.claim_id,
|
||||
claim, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
if signing_channel is not None:
|
||||
updated_claim.sign(signing_channel, b'placeholder txid:nout')
|
||||
else:
|
||||
updated_claim.clear_signature()
|
||||
return cls.create(
|
||||
[Input.spend(previous_claim)], [updated_claim], funding_accounts, change_account, sign=False
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
return cls.create([], [support_output], funding_accounts, change_account)
|
||||
return cls.create([], [claim_output], funding_accounts, change_account)
|
||||
|
||||
@classmethod
|
||||
def purchase(cls, claim: Output, amount: int, merchant_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
claim_output = Output.purchase_claim_pubkey_hash(
|
||||
amount, claim.claim_id, ledger.address_to_hash160(merchant_address)
|
||||
)
|
||||
return cls.create([], [claim_output], funding_accounts, change_account)
|
||||
|
||||
@classmethod
|
||||
def update(cls, previous_claim: Output, meta: ClaimDict, amount: int, holding_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
updated_claim = Output.pay_update_claim_pubkey_hash(
|
||||
amount, previous_claim.claim_name, previous_claim.claim_id,
|
||||
meta.serialized, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
return cls.create([Input.spend(previous_claim)], [updated_claim], funding_accounts, change_account)
|
||||
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: bytes,
|
||||
funding_accounts: List[Account], change_account: Account):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
return cls.create([], [output], funding_accounts, change_account)
|
||||
|
||||
@classmethod
|
||||
def abandon(cls, claims: Iterable[Output], funding_accounts: Iterable[Account], change_account: Account):
|
||||
return cls.create([Input.spend(txo) for txo in claims], [], funding_accounts, change_account)
|
||||
|
||||
@property
|
||||
def my_inputs(self):
|
||||
for txi in self.inputs:
|
||||
@@ -1,5 +1,6 @@
|
||||
build:
|
||||
rm types/v2/* -rf
|
||||
touch types/v2/__init__.py
|
||||
cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py
|
||||
rm proto/* -rf
|
||||
cd proto && protoc --python_out=. -I ../../../../types/proto/ ../../../../types/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i proto/*.py
|
||||
touch proto/__init__.py
|
||||
|
||||
@@ -1 +1,13 @@
|
||||
from .claim import Claim
|
||||
import os
|
||||
|
||||
__version__ = "0.0.16"
|
||||
|
||||
BLOCKCHAIN_NAME_ENVVAR = "LBRYSCHEMA_BLOCKCHAIN_NAME"
|
||||
if BLOCKCHAIN_NAME_ENVVAR in os.environ:
|
||||
if os.environ[BLOCKCHAIN_NAME_ENVVAR] in ['lbrycrd_main', 'lbrycrd_regtest',
|
||||
'lbrycrd_testnet']:
|
||||
BLOCKCHAIN_NAME = os.environ[BLOCKCHAIN_NAME_ENVVAR]
|
||||
else:
|
||||
raise OSError("invalid blockchain name: %s" % os.environ[BLOCKCHAIN_NAME_ENVVAR])
|
||||
else:
|
||||
BLOCKCHAIN_NAME = "lbrycrd_main"
|
||||
@@ -0,0 +1,64 @@
|
||||
import lbrynet.schema
|
||||
from lbrynet.schema.base import b58encode, b58decode, validate_b58_checksum
|
||||
from lbrynet.schema.hashing import double_sha256, hash160
|
||||
from lbrynet.schema.error import InvalidAddress
|
||||
from lbrynet.schema.schema import ADDRESS_LENGTH, ADDRESS_PREFIXES, PUBKEY_ADDRESS, SCRIPT_ADDRESS
|
||||
|
||||
|
||||
def validate_address_length(addr_bytes):
|
||||
if len(addr_bytes) != ADDRESS_LENGTH:
|
||||
raise InvalidAddress("Invalid address length: %i" % len(addr_bytes))
|
||||
|
||||
|
||||
def validate_address_prefix(addr_bytes):
|
||||
prefix = addr_bytes[0]
|
||||
if prefix not in ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME].values():
|
||||
raise InvalidAddress("Invalid address prefix: %.2X" % prefix)
|
||||
|
||||
|
||||
def validate_lbrycrd_address_bytes(addr_bytes):
|
||||
validate_address_length(addr_bytes)
|
||||
validate_address_prefix(addr_bytes)
|
||||
validate_b58_checksum(addr_bytes)
|
||||
return addr_bytes
|
||||
|
||||
|
||||
def decode_address(v):
|
||||
"""decode and validate a b58 address"""
|
||||
return validate_lbrycrd_address_bytes(b58decode(v))
|
||||
|
||||
|
||||
def encode_address(addr_bytes):
|
||||
"""validate and encode an address as b58"""
|
||||
v = validate_lbrycrd_address_bytes(addr_bytes)
|
||||
return b58encode(v)
|
||||
|
||||
|
||||
def hash_160_bytes_to_address(h160, addrtype=PUBKEY_ADDRESS):
|
||||
if addrtype == PUBKEY_ADDRESS:
|
||||
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS])
|
||||
elif addrtype == SCRIPT_ADDRESS:
|
||||
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][SCRIPT_ADDRESS])
|
||||
else:
|
||||
raise Exception("Invalid address prefix")
|
||||
return b58encode(prefix + h160 + double_sha256(prefix + h160)[0:4])
|
||||
|
||||
|
||||
def public_key_to_address(public_key):
|
||||
return hash_160_bytes_to_address(hash160(public_key))
|
||||
|
||||
|
||||
def address_to_hash_160(addr):
|
||||
bytes = decode_address(addr)
|
||||
prefix, pubkey_bytes, addr_checksum = bytes[0], bytes[1:21], bytes[21:]
|
||||
if prefix == chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS]):
|
||||
return PUBKEY_ADDRESS, pubkey_bytes
|
||||
return SCRIPT_ADDRESS, pubkey_bytes
|
||||
|
||||
|
||||
def is_address(addr):
|
||||
try:
|
||||
addr_bytes = decode_address(addr)
|
||||
return True
|
||||
except InvalidAddress:
|
||||
return False
|
||||
@@ -1,512 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import hashlib
|
||||
from typing import Tuple, List
|
||||
from string import ascii_letters
|
||||
from decimal import Decimal, ROUND_UP
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from torba.client.hash import Base58
|
||||
from torba.client.constants import COIN
|
||||
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
from lbrynet.schema.base import Metadata, BaseMessageList
|
||||
from lbrynet.schema.types.v2.claim_pb2 import (
|
||||
Fee as FeeMessage,
|
||||
Location as LocationMessage,
|
||||
Language as LanguageMessage
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_sha256_file_hash(file_path):
|
||||
sha256 = hashlib.sha256()
|
||||
with open(file_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(128 * sha256.block_size), b''):
|
||||
sha256.update(chunk)
|
||||
return sha256.digest()
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def width(self) -> int:
|
||||
return self.message.width
|
||||
|
||||
@width.setter
|
||||
def width(self, width: int):
|
||||
self.message.width = width
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.message.height
|
||||
|
||||
@height.setter
|
||||
def height(self, height: int):
|
||||
self.message.height = height
|
||||
|
||||
@property
|
||||
def dimensions(self) -> Tuple[int, int]:
|
||||
return self.width, self.height
|
||||
|
||||
@dimensions.setter
|
||||
def dimensions(self, dimensions: Tuple[int, int]):
|
||||
self.message.width, self.message.height = dimensions
|
||||
|
||||
def _extract(self, file_metadata, field):
|
||||
try:
|
||||
setattr(self, field, file_metadata.getValues(field)[0])
|
||||
except:
|
||||
log.exception(f'Could not extract {field} from file metadata.')
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None):
|
||||
if height is not None:
|
||||
self.height = height
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'height')
|
||||
|
||||
if width is not None:
|
||||
self.width = width
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'width')
|
||||
|
||||
|
||||
class Playable(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return self.message.duration
|
||||
|
||||
@duration.setter
|
||||
def duration(self, duration: int):
|
||||
self.message.duration = duration
|
||||
|
||||
def update(self, file_metadata=None, duration=None):
|
||||
if duration is not None:
|
||||
self.duration = duration
|
||||
elif file_metadata:
|
||||
try:
|
||||
self.duration = file_metadata.getValues('duration')[0].seconds
|
||||
except:
|
||||
log.exception('Could not extract duration from file metadata.')
|
||||
|
||||
|
||||
class Image(Dimmensional):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Audio(Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Video(Dimmensional, Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None, duration=None):
|
||||
Dimmensional.update(self, file_metadata, height, width)
|
||||
Playable.update(self, file_metadata, duration)
|
||||
|
||||
|
||||
class Source(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_path=None):
|
||||
if file_path is not None:
|
||||
self.name = os.path.basename(file_path)
|
||||
self.media_type, stream_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
self.size = os.path.getsize(file_path)
|
||||
if self.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
self.file_hash_bytes = calculate_sha256_file_hash(file_path)
|
||||
return stream_type
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.message.name
|
||||
|
||||
@name.setter
|
||||
def name(self, name: str):
|
||||
self.message.name = name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.message.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size: int):
|
||||
self.message.size = size
|
||||
|
||||
@property
|
||||
def media_type(self) -> str:
|
||||
return self.message.media_type
|
||||
|
||||
@media_type.setter
|
||||
def media_type(self, media_type: str):
|
||||
self.message.media_type = media_type
|
||||
|
||||
@property
|
||||
def file_hash(self) -> str:
|
||||
return hexlify(self.message.hash).decode()
|
||||
|
||||
@file_hash.setter
|
||||
def file_hash(self, file_hash: str):
|
||||
self.message.hash = unhexlify(file_hash.encode())
|
||||
|
||||
@property
|
||||
def file_hash_bytes(self) -> bytes:
|
||||
return self.message.hash
|
||||
|
||||
@file_hash_bytes.setter
|
||||
def file_hash_bytes(self, file_hash_bytes: bytes):
|
||||
self.message.hash = file_hash_bytes
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return hexlify(self.message.sd_hash).decode()
|
||||
|
||||
@sd_hash.setter
|
||||
def sd_hash(self, sd_hash: str):
|
||||
self.message.sd_hash = unhexlify(sd_hash.encode())
|
||||
|
||||
@property
|
||||
def sd_hash_bytes(self) -> bytes:
|
||||
return self.message.sd_hash
|
||||
|
||||
@sd_hash_bytes.setter
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.message.url
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str):
|
||||
self.message.url = url
|
||||
|
||||
|
||||
class Fee(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, address: str = None, currency: str = None, amount=None):
|
||||
if address is not None:
|
||||
self.address = address
|
||||
if currency is not None and amount is not None:
|
||||
currency = currency.lower()
|
||||
assert currency in ('lbc', 'btc', 'usd'), f'Unknown currency type: {currency}'
|
||||
setattr(self, currency, Decimal(amount))
|
||||
|
||||
@property
|
||||
def currency(self) -> str:
|
||||
return FeeMessage.Currency.Name(self.message.currency)
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
return Base58.encode(self.message.address)
|
||||
|
||||
@address.setter
|
||||
def address(self, address: str):
|
||||
self.message.address = Base58.decode(address)
|
||||
|
||||
@property
|
||||
def address_bytes(self) -> bytes:
|
||||
return self.message.address
|
||||
|
||||
@address_bytes.setter
|
||||
def address_bytes(self, address: bytes):
|
||||
self.message.address = address
|
||||
|
||||
@property
|
||||
def amount(self) -> Decimal:
|
||||
if self.currency == 'LBC':
|
||||
return self.lbc
|
||||
if self.currency == 'BTC':
|
||||
return self.btc
|
||||
if self.currency == 'USD':
|
||||
return self.usd
|
||||
|
||||
DEWIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def lbc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('LBC can only be returned for LBC fees.')
|
||||
return Decimal(self.message.amount / self.DEWIES)
|
||||
|
||||
@lbc.setter
|
||||
def lbc(self, amount: Decimal):
|
||||
self.dewies = int(amount * self.DEWIES)
|
||||
|
||||
@property
|
||||
def dewies(self) -> int:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('Dewies can only be returned for LBC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@dewies.setter
|
||||
def dewies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.LBC
|
||||
|
||||
SATOSHIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def btc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('BTC can only be returned for BTC fees.')
|
||||
return Decimal(self.message.amount / self.SATOSHIES)
|
||||
|
||||
@btc.setter
|
||||
def btc(self, amount: Decimal):
|
||||
self.satoshis = int(amount * self.SATOSHIES)
|
||||
|
||||
@property
|
||||
def satoshis(self) -> int:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('Satoshies can only be returned for BTC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@satoshis.setter
|
||||
def satoshis(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.BTC
|
||||
|
||||
PENNIES = Decimal('100.0')
|
||||
PENNY = Decimal('0.01')
|
||||
|
||||
@property
|
||||
def usd(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('USD can only be returned for USD fees.')
|
||||
return Decimal(self.message.amount / self.PENNIES)
|
||||
|
||||
@usd.setter
|
||||
def usd(self, amount: Decimal):
|
||||
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
|
||||
|
||||
@property
|
||||
def pennies(self) -> int:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('Pennies can only be returned for USD fees.')
|
||||
return self.message.amount
|
||||
|
||||
@pennies.setter
|
||||
def pennies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.USD
|
||||
|
||||
|
||||
class ClaimReference(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
|
||||
@claim_id.setter
|
||||
def claim_id(self, claim_id: str):
|
||||
self.claim_hash = unhexlify(claim_id)[::-1]
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
return self.message.claim_hash
|
||||
|
||||
@claim_hash.setter
|
||||
def claim_hash(self, claim_hash: bytes):
|
||||
self.message.claim_hash = claim_hash
|
||||
|
||||
|
||||
class ClaimList(BaseMessageList[ClaimReference]):
|
||||
|
||||
__slots__ = ()
|
||||
item_class = ClaimReference
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message.claim_references
|
||||
|
||||
def append(self, value):
|
||||
self.add().claim_id = value
|
||||
|
||||
@property
|
||||
def ids(self) -> List[str]:
|
||||
return [c.claim_id for c in self]
|
||||
|
||||
|
||||
class Language(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def langtag(self) -> str:
|
||||
langtag = []
|
||||
if self.language:
|
||||
langtag.append(self.language)
|
||||
if self.script:
|
||||
langtag.append(self.script)
|
||||
if self.region:
|
||||
langtag.append(self.region)
|
||||
return '-'.join(langtag)
|
||||
|
||||
@langtag.setter
|
||||
def langtag(self, langtag: str):
|
||||
parts = langtag.split('-')
|
||||
self.language = parts.pop(0)
|
||||
if parts and len(parts[0]) == 4:
|
||||
self.script = parts.pop(0)
|
||||
if parts and len(parts[0]) == 2:
|
||||
self.region = parts.pop(0)
|
||||
assert not parts, f"Failed to parse language tag: {langtag}"
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
if self.message.language:
|
||||
return LanguageMessage.Language.Name(self.message.language)
|
||||
|
||||
@language.setter
|
||||
def language(self, language: str):
|
||||
self.message.language = LanguageMessage.Language.Value(language)
|
||||
|
||||
@property
|
||||
def script(self) -> str:
|
||||
if self.message.script:
|
||||
return LanguageMessage.Script.Name(self.message.script)
|
||||
|
||||
@script.setter
|
||||
def script(self, script: str):
|
||||
self.message.script = LanguageMessage.Script.Value(script)
|
||||
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
return LocationMessage.Country.Name(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
__slots__ = ()
|
||||
item_class = Language
|
||||
|
||||
def append(self, value: str):
|
||||
self.add().langtag = value
|
||||
|
||||
|
||||
class Location(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def from_value(self, value):
|
||||
if isinstance(value, str) and value.startswith('{'):
|
||||
value = json.loads(value)
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, val in value.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
elif isinstance(value, str):
|
||||
parts = value.split(':')
|
||||
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
|
||||
country = parts and parts.pop(0)
|
||||
if country:
|
||||
self.country = country
|
||||
state = parts and parts.pop(0)
|
||||
if state:
|
||||
self.state = state
|
||||
city = parts and parts.pop(0)
|
||||
if city:
|
||||
self.city = city
|
||||
code = parts and parts.pop(0)
|
||||
if code:
|
||||
self.code = code
|
||||
latitude = parts and parts.pop(0)
|
||||
if latitude:
|
||||
self.latitude = latitude
|
||||
longitude = parts and parts.pop(0)
|
||||
if longitude:
|
||||
self.longitude = longitude
|
||||
|
||||
else:
|
||||
raise ValueError(f'Could not parse country value: {value}')
|
||||
|
||||
@property
|
||||
def country(self) -> str:
|
||||
if self.message.country:
|
||||
return LocationMessage.Country.Name(self.message.country)
|
||||
|
||||
@country.setter
|
||||
def country(self, country: str):
|
||||
self.message.country = LocationMessage.Country.Value(country)
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
return self.message.state
|
||||
|
||||
@state.setter
|
||||
def state(self, state: str):
|
||||
self.message.state = state
|
||||
|
||||
@property
|
||||
def city(self) -> str:
|
||||
return self.message.city
|
||||
|
||||
@city.setter
|
||||
def city(self, city: str):
|
||||
self.message.city = city
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self.message.code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str):
|
||||
self.message.code = code
|
||||
|
||||
GPS_PRECISION = Decimal('10000000')
|
||||
|
||||
@property
|
||||
def latitude(self) -> str:
|
||||
if self.message.latitude:
|
||||
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
|
||||
|
||||
@latitude.setter
|
||||
def latitude(self, latitude: str):
|
||||
latitude = Decimal(latitude)
|
||||
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
|
||||
self.message.latitude = int(latitude * self.GPS_PRECISION)
|
||||
|
||||
@property
|
||||
def longitude(self) -> str:
|
||||
if self.message.longitude:
|
||||
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
|
||||
|
||||
@longitude.setter
|
||||
def longitude(self, longitude: str):
|
||||
longitude = Decimal(longitude)
|
||||
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
|
||||
self.message.longitude = int(longitude * self.GPS_PRECISION)
|
||||
|
||||
|
||||
class LocationList(BaseMessageList[Location]):
|
||||
__slots__ = ()
|
||||
item_class = Location
|
||||
|
||||
def append(self, value):
|
||||
self.add().from_value(value)
|
||||
+72
-103
@@ -1,121 +1,90 @@
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterator, TypeVar, Generic
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from lbrynet.schema.schema import ADDRESS_CHECKSUM_LENGTH
|
||||
from lbrynet.schema.hashing import double_sha256
|
||||
from lbrynet.schema.error import InvalidAddress
|
||||
|
||||
|
||||
class Signable:
|
||||
|
||||
__slots__ = (
|
||||
'message', 'version', 'signature',
|
||||
'signature_type', 'unsigned_payload', 'signing_channel_hash'
|
||||
)
|
||||
|
||||
message_class = None
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.message_class()
|
||||
self.version = 2
|
||||
self.signature = None
|
||||
self.signature_type = 'SECP256k1'
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
def clear_signature(self):
|
||||
self.signature = None
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
@property
|
||||
def signing_channel_id(self):
|
||||
return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
|
||||
|
||||
@signing_channel_id.setter
|
||||
def signing_channel_id(self, channel_id: str):
|
||||
self.signing_channel_hash = unhexlify(channel_id)[::-1]
|
||||
|
||||
@property
|
||||
def is_signed(self):
|
||||
return self.signature is not None
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message)
|
||||
|
||||
def to_message_bytes(self) -> bytes:
|
||||
return self.message.SerializeToString()
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
pieces = bytearray()
|
||||
if self.is_signed:
|
||||
pieces.append(1)
|
||||
pieces.extend(self.signing_channel_hash)
|
||||
pieces.extend(self.signature)
|
||||
else:
|
||||
pieces.append(0)
|
||||
pieces.extend(self.to_message_bytes())
|
||||
return bytes(pieces)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes):
|
||||
signable = cls()
|
||||
if data[0] == 0:
|
||||
signable.message.ParseFromString(data[1:])
|
||||
elif data[0] == 1:
|
||||
signable.signing_channel_hash = data[1:21]
|
||||
signable.signature = data[21:85]
|
||||
signable.message.ParseFromString(data[85:])
|
||||
else:
|
||||
raise DecodeError('Could not determine message format version.')
|
||||
return signable
|
||||
|
||||
def __len__(self):
|
||||
return len(self.to_bytes())
|
||||
|
||||
def __bytes__(self):
|
||||
return self.to_bytes()
|
||||
alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
|
||||
|
||||
class Metadata:
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
def scrub_input(v):
|
||||
if isinstance(v, str) and not isinstance(v, bytes):
|
||||
v = v.encode('ascii')
|
||||
return v
|
||||
|
||||
|
||||
I = TypeVar('I')
|
||||
def b58encode_int(i, default_one=True):
|
||||
'''Encode an integer using Base58'''
|
||||
if not i and default_one:
|
||||
return alphabet[0:1]
|
||||
string = b""
|
||||
while i:
|
||||
i, idx = divmod(i, 58)
|
||||
string = alphabet[idx:idx+1] + string
|
||||
return string
|
||||
|
||||
|
||||
class BaseMessageList(Metadata, Generic[I]):
|
||||
def b58encode(v):
|
||||
'''Encode a string using Base58'''
|
||||
|
||||
__slots__ = ()
|
||||
v = scrub_input(v)
|
||||
|
||||
item_class = None
|
||||
nPad = len(v)
|
||||
v = v.lstrip(b'\0')
|
||||
nPad -= len(v)
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message
|
||||
p, acc = 1, 0
|
||||
for c in reversed(v):
|
||||
acc += p * c
|
||||
p = p << 8
|
||||
|
||||
def add(self) -> I:
|
||||
return self.item_class(self._message.add())
|
||||
result = b58encode_int(acc, default_one=False)
|
||||
|
||||
def extend(self, values: List[str]):
|
||||
for value in values:
|
||||
self.append(value)
|
||||
return alphabet[0:1] * nPad + result
|
||||
|
||||
def append(self, value: str):
|
||||
raise NotImplemented
|
||||
|
||||
def __len__(self):
|
||||
return len(self._message)
|
||||
def b58decode_int(v):
|
||||
'''Decode a Base58 encoded string as an integer'''
|
||||
|
||||
def __iter__(self) -> Iterator[I]:
|
||||
for item in self._message:
|
||||
yield self.item_class(item)
|
||||
v = scrub_input(v)
|
||||
|
||||
def __getitem__(self, item) -> I:
|
||||
return self.item_class(self._message[item])
|
||||
decimal = 0
|
||||
for char in v:
|
||||
decimal = decimal * 58 + alphabet.index(char)
|
||||
return decimal
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._message[key]
|
||||
|
||||
def b58decode(v):
|
||||
'''Decode a Base58 encoded string'''
|
||||
|
||||
v = scrub_input(v)
|
||||
|
||||
origlen = len(v)
|
||||
v = v.lstrip(alphabet[0:1])
|
||||
newlen = len(v)
|
||||
|
||||
acc = b58decode_int(v)
|
||||
|
||||
result = []
|
||||
while acc > 0:
|
||||
acc, mod = divmod(acc, 256)
|
||||
result.append(mod)
|
||||
|
||||
return b'\0' * (origlen - newlen) + bytes(reversed(result))
|
||||
|
||||
|
||||
def validate_b58_checksum(addr_bytes):
|
||||
addr_without_checksum = addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
|
||||
addr_checksum = addr_bytes[-ADDRESS_CHECKSUM_LENGTH:]
|
||||
if double_sha256(addr_without_checksum)[:ADDRESS_CHECKSUM_LENGTH] != addr_checksum:
|
||||
raise InvalidAddress("Invalid address checksum")
|
||||
|
||||
|
||||
def b58decode_strip_checksum(v):
|
||||
addr_bytes = b58decode(v)
|
||||
validate_b58_checksum(addr_bytes)
|
||||
return addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
|
||||
|
||||
|
||||
def b58encode_with_checksum(addr_bytes):
|
||||
addr_checksum = double_sha256(addr_bytes)[:ADDRESS_CHECKSUM_LENGTH]
|
||||
return b58encode(addr_bytes + addr_checksum)
|
||||
+171
-351
@@ -1,378 +1,198 @@
|
||||
import logging
|
||||
from typing import List
|
||||
from binascii import hexlify, unhexlify
|
||||
import json
|
||||
import binascii
|
||||
from google.protobuf import json_format # pylint: disable=no-name-in-module
|
||||
from google.protobuf.message import DecodeError as DecodeError_pb # pylint: disable=no-name-in-module,import-error
|
||||
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from google.protobuf.message import DecodeError
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
from hachoir.parser import createParser as binary_file_parser
|
||||
from hachoir.metadata import extractMetadata as binary_file_metadata
|
||||
from collections import OrderedDict
|
||||
|
||||
from lbrynet.schema import compat
|
||||
from lbrynet.schema.base import Signable
|
||||
from lbrynet.schema.mime_types import guess_media_type, guess_stream_type
|
||||
from lbrynet.schema.attrs import (
|
||||
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
|
||||
LanguageList, LocationList, ClaimList, ClaimReference
|
||||
)
|
||||
from lbrynet.schema.types.v2.claim_pb2 import Claim as ClaimMessage
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.proto import claim_pb2
|
||||
from lbrynet.schema.signature import Signature
|
||||
from lbrynet.schema.validator import get_validator
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
|
||||
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
|
||||
hachoir_log.use_print = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Claim(Signable):
|
||||
|
||||
STREAM = 'stream'
|
||||
CHANNEL = 'channel'
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = 'version',
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
class ClaimDict(OrderedDict):
|
||||
def __init__(self, claim_dict=None, detached_signature: Signature=None):
|
||||
if isinstance(claim_dict, claim_pb2.Claim):
|
||||
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
|
||||
(self.__class__.__name__, self.__class__.__name__))
|
||||
self.detached_signature = detached_signature
|
||||
OrderedDict.__init__(self, claim_dict or [])
|
||||
|
||||
@property
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
def protobuf_dict(self):
|
||||
"""Claim dictionary using base64 to represent bytes"""
|
||||
|
||||
def get_message(self, type_name):
|
||||
message = getattr(self.message, type_name)
|
||||
if self.claim_type is None:
|
||||
message.SetInParent()
|
||||
if self.claim_type != type_name:
|
||||
raise ValueError(f'Claim is not a {type_name}.')
|
||||
return message
|
||||
return json.loads(json_format.MessageToJson(self.protobuf, True))
|
||||
|
||||
@property
|
||||
def protobuf(self):
|
||||
"""Claim message object"""
|
||||
|
||||
return Claim.load(self)
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
"""Serialized Claim protobuf"""
|
||||
if self.detached_signature:
|
||||
return self.detached_signature.serialized
|
||||
return self.protobuf.SerializeToString()
|
||||
|
||||
@property
|
||||
def serialized_no_signature(self):
|
||||
"""Serialized Claim protobuf without publisherSignature field"""
|
||||
claim = self.protobuf
|
||||
claim.ClearField("publisherSignature")
|
||||
return ClaimDict.load_protobuf(claim).serialized
|
||||
|
||||
@property
|
||||
def has_signature(self):
|
||||
return self.protobuf.HasField("publisherSignature") or (
|
||||
self.detached_signature and self.detached_signature.raw_signature
|
||||
)
|
||||
|
||||
@property
|
||||
def is_certificate(self):
|
||||
claim = self.protobuf
|
||||
return CLAIM_TYPE_NAMES[claim.claimType] == "certificate"
|
||||
|
||||
@property
|
||||
def is_stream(self):
|
||||
return self.claim_type == self.STREAM
|
||||
claim = self.protobuf
|
||||
return CLAIM_TYPE_NAMES[claim.claimType] == "stream"
|
||||
|
||||
@property
|
||||
def stream(self) -> 'Stream':
|
||||
return Stream(self)
|
||||
def source_hash(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
return binascii.hexlify(claim.stream.source.source)
|
||||
|
||||
@property
|
||||
def is_channel(self):
|
||||
return self.claim_type == self.CHANNEL
|
||||
def has_fee(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
if claim.stream.metadata.HasField("fee"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def channel(self) -> 'Channel':
|
||||
return Channel(self)
|
||||
def source_fee(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
if claim.stream.metadata.HasField("fee"):
|
||||
return Fee.load_protobuf(claim.stream.metadata.fee)
|
||||
return None
|
||||
|
||||
@property
|
||||
def is_repost(self):
|
||||
return self.claim_type == self.REPOST
|
||||
def certificate_id(self) -> str:
|
||||
if self.protobuf.HasField("publisherSignature"):
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
|
||||
if self.detached_signature and self.detached_signature.certificate_id:
|
||||
return binascii.hexlify(self.detached_signature.certificate_id).decode()
|
||||
|
||||
@property
|
||||
def repost(self) -> 'Repost':
|
||||
return Repost(self)
|
||||
def signature(self):
|
||||
if not self.has_signature:
|
||||
return None
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.signature)
|
||||
|
||||
@property
|
||||
def is_collection(self):
|
||||
return self.claim_type == self.COLLECTION
|
||||
def protobuf_len(self):
|
||||
"""Length of serialized string"""
|
||||
|
||||
return self.protobuf.ByteSize()
|
||||
|
||||
@property
|
||||
def collection(self) -> 'Collection':
|
||||
return Collection(self)
|
||||
def json_len(self):
|
||||
"""Length of json encoded string"""
|
||||
|
||||
return len(json.dumps(self.claim_dict))
|
||||
|
||||
@property
|
||||
def claim_dict(self):
|
||||
"""Claim dictionary with bytes represented as hex and base58"""
|
||||
|
||||
return dict(encode_fields(self, self.detached_signature))
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes) -> 'Claim':
|
||||
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
|
||||
"""
|
||||
Load a ClaimDict from a dictionary with base64 encoded bytes
|
||||
(as returned by the protobuf json formatter)
|
||||
"""
|
||||
|
||||
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, protobuf_claim, detached_signature=None):
|
||||
"""Load ClaimDict from a protobuf Claim message"""
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_dict(cls, claim_dict):
|
||||
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
|
||||
try:
|
||||
return super().from_bytes(data)
|
||||
except DecodeError:
|
||||
claim = cls()
|
||||
if data[0] == ord('{'):
|
||||
claim.version = 0
|
||||
compat.from_old_json_schema(claim, data)
|
||||
elif data[0] not in (0, 1):
|
||||
claim.version = 1
|
||||
compat.from_types_v1(claim, data)
|
||||
else:
|
||||
raise
|
||||
return claim
|
||||
|
||||
|
||||
class BaseClaim:
|
||||
|
||||
__slots__ = 'claim', 'message'
|
||||
|
||||
claim_type = None
|
||||
object_fields = 'thumbnail',
|
||||
repeat_fields = 'tags', 'languages', 'locations'
|
||||
|
||||
def __init__(self, claim: Claim = None):
|
||||
self.claim = claim or Claim()
|
||||
self.message = self.claim.get_message(self.claim_type)
|
||||
|
||||
def to_dict(self):
|
||||
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
|
||||
claim.update(claim.pop(self.claim_type))
|
||||
if 'languages' in claim:
|
||||
claim['languages'] = self.langtags
|
||||
return claim
|
||||
|
||||
def update(self, **kwargs):
|
||||
for key in list(kwargs):
|
||||
for field in self.object_fields:
|
||||
if key.startswith(f'{field}_'):
|
||||
attr = getattr(self, field)
|
||||
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
|
||||
continue
|
||||
|
||||
for l in self.repeat_fields:
|
||||
field = getattr(self, l)
|
||||
if kwargs.pop(f'clear_{l}', False):
|
||||
del field[:]
|
||||
items = kwargs.pop(l, None)
|
||||
if items is not None:
|
||||
if isinstance(items, str):
|
||||
field.append(items)
|
||||
elif isinstance(items, list):
|
||||
field.extend(items)
|
||||
else:
|
||||
raise ValueError(f"Unknown {l} value: {items}")
|
||||
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.claim.message.title
|
||||
|
||||
@title.setter
|
||||
def title(self, title: str):
|
||||
self.claim.message.title = title
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.claim.message.description
|
||||
|
||||
@description.setter
|
||||
def description(self, description: str):
|
||||
self.claim.message.description = description
|
||||
|
||||
@property
|
||||
def thumbnail(self) -> Source:
|
||||
return Source(self.claim.message.thumbnail)
|
||||
|
||||
@property
|
||||
def tags(self) -> List:
|
||||
return self.claim.message.tags
|
||||
|
||||
@property
|
||||
def languages(self) -> LanguageList:
|
||||
return LanguageList(self.claim.message.languages)
|
||||
|
||||
@property
|
||||
def langtags(self) -> List[str]:
|
||||
return [l.langtag for l in self.languages]
|
||||
|
||||
@property
|
||||
def locations(self) -> LocationList:
|
||||
return LocationList(self.claim.message.locations)
|
||||
|
||||
|
||||
class Stream(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.STREAM
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('source',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'source' in claim:
|
||||
if 'hash' in claim['source']:
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
fee = claim.get('fee', {})
|
||||
if 'address' in fee:
|
||||
fee['address'] = self.fee.address
|
||||
if 'amount' in fee:
|
||||
fee['amount'] = self.fee.amount
|
||||
return claim
|
||||
|
||||
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
|
||||
self.fee.update(
|
||||
kwargs.pop('fee_address', None),
|
||||
kwargs.pop('fee_currency', None),
|
||||
kwargs.pop('fee_amount', None)
|
||||
)
|
||||
|
||||
if 'sd_hash' in kwargs:
|
||||
self.source.sd_hash = kwargs.pop('sd_hash')
|
||||
|
||||
stream_type = None
|
||||
if file_path is not None:
|
||||
stream_type = self.source.update(file_path=file_path)
|
||||
elif self.source.name:
|
||||
self.source.media_type, stream_type = guess_media_type(self.source.name)
|
||||
elif self.source.media_type:
|
||||
stream_type = guess_stream_type(self.source.media_type)
|
||||
|
||||
if stream_type in ('image', 'video', 'audio'):
|
||||
media = getattr(self, stream_type)
|
||||
media_args = {'file_metadata': None}
|
||||
try:
|
||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||
except:
|
||||
log.exception('Could not read file metadata.')
|
||||
if isinstance(media, Playable):
|
||||
media_args['duration'] = duration
|
||||
if isinstance(media, Dimmensional):
|
||||
media_args['height'] = height
|
||||
media_args['width'] = width
|
||||
media.update(**media_args)
|
||||
|
||||
super().update(**kwargs)
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
return self.message.author
|
||||
|
||||
@author.setter
|
||||
def author(self, author: str):
|
||||
self.message.author = author
|
||||
|
||||
@property
|
||||
def license(self) -> str:
|
||||
return self.message.license
|
||||
|
||||
@license.setter
|
||||
def license(self, license: str):
|
||||
self.message.license = license
|
||||
|
||||
@property
|
||||
def license_url(self) -> str:
|
||||
return self.message.license_url
|
||||
|
||||
@license_url.setter
|
||||
def license_url(self, license_url: str):
|
||||
self.message.license_url = license_url
|
||||
|
||||
@property
|
||||
def release_time(self) -> int:
|
||||
return self.message.release_time
|
||||
|
||||
@release_time.setter
|
||||
def release_time(self, release_time: int):
|
||||
self.message.release_time = release_time
|
||||
|
||||
@property
|
||||
def fee(self) -> Fee:
|
||||
return Fee(self.message.fee)
|
||||
|
||||
@property
|
||||
def has_fee(self) -> bool:
|
||||
return self.message.HasField('fee')
|
||||
|
||||
@property
|
||||
def source(self) -> Source:
|
||||
return Source(self.message.source)
|
||||
|
||||
@property
|
||||
def image(self) -> Image:
|
||||
return Image(self.message.image)
|
||||
|
||||
@property
|
||||
def video(self) -> Video:
|
||||
return Video(self.message.video)
|
||||
|
||||
@property
|
||||
def audio(self) -> Audio:
|
||||
return Audio(self.message.audio)
|
||||
|
||||
|
||||
class Channel(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.CHANNEL
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('cover',)
|
||||
repeat_fields = BaseClaim.repeat_fields + ('featured',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
claim['public_key'] = self.public_key
|
||||
if 'featured' in claim:
|
||||
claim['featured'] = self.featured.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.message.public_key).decode()
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
self.message.public_key = unhexlify(sd_public_key.encode())
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
return self.message.public_key
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
self.message.public_key = public_key
|
||||
|
||||
@property
|
||||
def email(self) -> str:
|
||||
return self.message.email
|
||||
|
||||
@email.setter
|
||||
def email(self, email: str):
|
||||
self.message.email = email
|
||||
|
||||
@property
|
||||
def website_url(self) -> str:
|
||||
return self.message.website_url
|
||||
|
||||
@website_url.setter
|
||||
def website_url(self, website_url: str):
|
||||
self.message.website_url = website_url
|
||||
|
||||
@property
|
||||
def cover(self) -> Source:
|
||||
return Source(self.message.cover)
|
||||
|
||||
@property
|
||||
def featured(self) -> ClaimList:
|
||||
return ClaimList(self.message.featured)
|
||||
|
||||
|
||||
class Repost(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.REPOST
|
||||
|
||||
@property
|
||||
def reference(self) -> ClaimReference:
|
||||
return ClaimReference(self.message)
|
||||
|
||||
|
||||
class Collection(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.COLLECTION
|
||||
|
||||
repeat_fields = BaseClaim.repeat_fields + ('claims',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'claim_references' in claim:
|
||||
claim['claim_references'] = self.claims.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def claims(self) -> ClaimList:
|
||||
return ClaimList(self.message)
|
||||
claim_dict, detached_signature = decode_fields(claim_dict)
|
||||
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
|
||||
except json_format.ParseError as err:
|
||||
raise DecodeError(str(err))
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
"""Load a ClaimDict from a serialized protobuf string"""
|
||||
detached_signature = Signature.flagged_parse(serialized)
|
||||
|
||||
temp_claim = claim_pb2.Claim()
|
||||
try:
|
||||
temp_claim.ParseFromString(detached_signature.payload)
|
||||
except DecodeError_pb:
|
||||
raise DecodeError(DecodeError_pb)
|
||||
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def generate_certificate(cls, private_key, curve=SECP256k1):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
return cls.load_protobuf(signer.certificate)
|
||||
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, force_detached=False):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, force_detached)
|
||||
return ClaimDict.load_protobuf(signed, signature)
|
||||
|
||||
def validate_signature(self, claim_address, certificate, name=None):
|
||||
if isinstance(certificate, ClaimDict):
|
||||
certificate = certificate.protobuf
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
|
||||
return validator.validate_claim_signature(self, claim_address, name)
|
||||
|
||||
def validate_private_key(self, private_key, certificate_id):
|
||||
certificate = self.protobuf
|
||||
if CLAIM_TYPE_NAMES[certificate.claimType] != "certificate":
|
||||
return
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, certificate_id)
|
||||
signing_key = validator.signing_key_from_pem(private_key)
|
||||
return validator.validate_private_key(signing_key)
|
||||
|
||||
def get_validator(self, certificate_id):
|
||||
"""
|
||||
Get a lbrynet.schema.validator.Validator object for a certificate claim
|
||||
|
||||
:param certificate_id: claim id of this certificate claim
|
||||
:return: None or lbrynet.schema.validator.Validator object
|
||||
"""
|
||||
|
||||
claim = self.protobuf
|
||||
if CLAIM_TYPE_NAMES[claim.claimType] != "certificate":
|
||||
return
|
||||
curve = CURVE_NAMES[claim.certificate.keyType]
|
||||
return get_validator(curve).load_from_certificate(claim, certificate_id)
|
||||
@@ -1,91 +0,0 @@
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
|
||||
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
|
||||
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
|
||||
|
||||
|
||||
def from_old_json_schema(claim, payload: bytes):
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except:
|
||||
raise DecodeError('Could not parse JSON.')
|
||||
stream = claim.stream
|
||||
stream.source.sd_hash = value['sources']['lbry_sd_hash']
|
||||
stream.source.media_type = value.get(
|
||||
'content_type', value.get('content-type', 'application/octet-stream'))
|
||||
stream.title = value.get('title', '')
|
||||
stream.description = value.get('description', '')
|
||||
if value.get('thumbnail', ''):
|
||||
stream.thumbnail.url = value.get('thumbnail', '')
|
||||
stream.author = value.get('author', '')
|
||||
stream.license = value.get('license', '')
|
||||
stream.license_url = value.get('license_url', '')
|
||||
language = value.get('language', '')
|
||||
if language:
|
||||
if language.lower() == 'english':
|
||||
language = 'en'
|
||||
try:
|
||||
stream.languages.append(language)
|
||||
except:
|
||||
pass
|
||||
if value.get('nsfw', False):
|
||||
stream.tags.append('mature')
|
||||
if "fee" in value:
|
||||
fee = value["fee"]
|
||||
currency = list(fee.keys())[0]
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee[currency]['amount'])
|
||||
else:
|
||||
raise DecodeError(f'Unknown currency: {currency}')
|
||||
stream.fee.address = fee[currency]['address']
|
||||
return claim
|
||||
|
||||
|
||||
def from_types_v1(claim, payload: bytes):
|
||||
old = OldClaimMessage()
|
||||
old.ParseFromString(payload)
|
||||
if old.claimType == 2:
|
||||
channel = claim.channel
|
||||
channel.public_key_bytes = old.certificate.publicKey
|
||||
else:
|
||||
stream = claim.stream
|
||||
stream.title = old.stream.metadata.title
|
||||
stream.description = old.stream.metadata.description
|
||||
stream.author = old.stream.metadata.author
|
||||
stream.license = old.stream.metadata.license
|
||||
stream.license_url = old.stream.metadata.licenseUrl
|
||||
stream.thumbnail.url = old.stream.metadata.thumbnail
|
||||
if old.stream.metadata.HasField('language'):
|
||||
stream.languages.add().message.language = old.stream.metadata.language
|
||||
stream.source.media_type = old.stream.source.contentType
|
||||
stream.source.sd_hash_bytes = old.stream.source.source
|
||||
if old.stream.metadata.nsfw:
|
||||
stream.tags.append('mature')
|
||||
if old.stream.metadata.HasField('fee'):
|
||||
fee = old.stream.metadata.fee
|
||||
stream.fee.address_bytes = fee.address
|
||||
currency = FeeMessage.Currency.Name(fee.currency)
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee.amount)
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee.amount)
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee.amount)
|
||||
else:
|
||||
raise DecodeError(f'Unsupported currency: {currency}')
|
||||
if old.HasField('publisherSignature'):
|
||||
sig = old.publisherSignature
|
||||
claim.signature = sig.signature
|
||||
claim.signature_type = KeyType.Name(sig.signatureType)
|
||||
claim.signing_channel_hash = sig.certificateId[::-1]
|
||||
old.ClearField("publisherSignature")
|
||||
claim.unsigned_payload = old.SerializeToString()
|
||||
return claim
|
||||
@@ -0,0 +1,61 @@
|
||||
import json
|
||||
import binascii
|
||||
from google.protobuf import json_format # pylint: disable=no-name-in-module
|
||||
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.legacy.migrate import migrate as schema_migrator
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
|
||||
|
||||
def migrate_json_claim_value(decoded_json):
|
||||
try:
|
||||
if 'fee' in decoded_json:
|
||||
old_fee = decoded_json['fee']
|
||||
if not old_fee[list(old_fee.keys())[0]]['amount']:
|
||||
del decoded_json['fee']
|
||||
return migrate_json_claim_value(decoded_json)
|
||||
except (TypeError, AttributeError, InvalidAddress):
|
||||
raise DecodeError("Failed to decode claim")
|
||||
try:
|
||||
pb_migrated = schema_migrator(decoded_json)
|
||||
return pb_migrated
|
||||
except json_format.ParseError as parse_error:
|
||||
raise DecodeError("Failed to parse protobuf: %s" % parse_error)
|
||||
except Exception as err:
|
||||
raise DecodeError("Failed to migrate claim: %s" % err)
|
||||
|
||||
|
||||
def smart_decode(claim_value):
|
||||
"""
|
||||
Decode a claim value
|
||||
|
||||
Try decoding claim protobuf, if this fails try decoding json and migrating it.
|
||||
If unable to decode or migrate, raise DecodeError
|
||||
"""
|
||||
|
||||
# if already decoded, return
|
||||
if isinstance(claim_value, ClaimDict):
|
||||
return claim_value
|
||||
elif isinstance(claim_value, dict):
|
||||
return ClaimDict.load_dict(claim_value)
|
||||
|
||||
try:
|
||||
claim_value = binascii.unhexlify(claim_value)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if claim_value[0] in ['{', ord('{')]:
|
||||
try:
|
||||
if isinstance(claim_value, bytes):
|
||||
claim_value = claim_value.decode()
|
||||
decoded_json = json.loads(claim_value)
|
||||
return migrate_json_claim_value(decoded_json)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
if isinstance(claim_value, str):
|
||||
claim_value = claim_value.encode()
|
||||
return ClaimDict.deserialize(claim_value)
|
||||
except (DecodeError, InvalidAddress, KeyError, TypeError):
|
||||
raise DecodeError(claim_value)
|
||||
@@ -0,0 +1,90 @@
|
||||
import base64, binascii
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.address import decode_address, encode_address
|
||||
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
|
||||
from lbrynet.schema.schema import SIGNATURE
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.signature import Signature
|
||||
|
||||
|
||||
def encode_fields(claim_dictionary, detached_signature: Signature):
|
||||
"""Encode bytes to hex and b58 for return by ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = encode_address(claim_value['metadata']['fee']['address'])
|
||||
except InvalidAddress as err:
|
||||
raise DecodeError("Invalid fee address: %s" % err)
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = claim_value["publicKey"]
|
||||
claim_value["publicKey"] = binascii.hexlify(public_key).decode()
|
||||
if SIGNATURE in claim_dictionary:
|
||||
encoded_sig = binascii.hexlify(claim_dictionary[SIGNATURE]['signature']).decode()
|
||||
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
elif detached_signature and detached_signature.raw_signature:
|
||||
claim_dictionary[SIGNATURE] = {
|
||||
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
|
||||
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
|
||||
}
|
||||
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
|
||||
|
||||
def decode_fields(claim_dictionary):
|
||||
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
|
||||
detached_signature = None
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.unhexlify(claim_value['source']['source'])
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = decode_address(claim_value['metadata']['fee']['address'])
|
||||
except InvalidAddress as err:
|
||||
raise DecodeError("Invalid fee address: %s" % err)
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = binascii.unhexlify(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
|
||||
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
|
||||
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
|
||||
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
|
||||
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
|
||||
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
|
||||
del claim_dictionary[SIGNATURE]
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary, detached_signature
|
||||
|
||||
|
||||
def decode_b64_fields(claim_dictionary):
|
||||
"""Decode b64 encoded bytes in protobuf generated dictionary to be given to ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = base64.b64decode(claim_value['source']['source'])
|
||||
if 'fee' in claim_value['metadata']:
|
||||
address = base64.b64decode(claim_value['metadata']['fee']['address'])
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = base64.b64decode(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary:
|
||||
encoded_sig = base64.b64decode(claim_dictionary[SIGNATURE]['signature'])
|
||||
encoded_cert_id = base64.b64decode(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
Loaded 100 of 209 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user