Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e01fbc6f39 |
No files matched your search
@@ -33,7 +33,9 @@ unsafe-load-any-extension=no
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
# extension-pkg-whitelist=
|
||||
extension-pkg-whitelist=
|
||||
miniupnpc,
|
||||
unqlite
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
Copyright (c) 2015-2018 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
build:
|
||||
python setup.py apk
|
||||
adb install -r lbry-0.1-service-debug.apk
|
||||
|
||||
.PHONY: build
|
||||
@@ -0,0 +1,49 @@
|
||||
import inspect
|
||||
from os.path import dirname, join
|
||||
|
||||
import sh
|
||||
from pythonforandroid.util import ensure_dir
|
||||
from pythonforandroid.toolchain import (
|
||||
Bootstrap, current_directory, info, info_main, shprint
|
||||
)
|
||||
|
||||
|
||||
class LBRYServiceBootstrap(Bootstrap):
|
||||
name = 'lbry-service'
|
||||
recipe_depends = ['genericndkbuild', 'python3']
|
||||
bootstrap_dir = dirname(__file__)
|
||||
|
||||
def get_common_dir(self):
|
||||
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
|
||||
|
||||
def run_distribute(self):
|
||||
info_main('# Creating Android project from build and {} bootstrap'.format(
|
||||
self.name))
|
||||
|
||||
info('This currently just copies the build stuff straight from the build dir.')
|
||||
shprint(sh.rm, '-rf', self.dist_dir)
|
||||
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
|
||||
with current_directory(self.dist_dir):
|
||||
with open('local.properties', 'w') as fileh:
|
||||
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
|
||||
|
||||
arch = self.ctx.archs[0]
|
||||
if len(self.ctx.archs) > 1:
|
||||
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
|
||||
info('Bootstrap running with arch {}'.format(arch))
|
||||
|
||||
with current_directory(self.dist_dir):
|
||||
info('Copying python distribution')
|
||||
|
||||
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
|
||||
self.distribute_aars(arch)
|
||||
self.distribute_javaclasses(self.ctx.javaclass_dir)
|
||||
|
||||
python_bundle_dir = join('_python_bundle', '_python_bundle')
|
||||
ensure_dir(python_bundle_dir)
|
||||
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
|
||||
join(self.dist_dir, python_bundle_dir), arch)
|
||||
|
||||
self.strip_libraries(arch)
|
||||
self.fry_eggs(site_packages_dir)
|
||||
super().run_distribute()
|
||||
@@ -0,0 +1,89 @@
|
||||
# prevent user to include invalid extensions
|
||||
*.apk
|
||||
*.pxd
|
||||
|
||||
# eggs
|
||||
*.egg-info
|
||||
|
||||
# unit test
|
||||
unittest/*
|
||||
|
||||
# python config
|
||||
config/makesetup
|
||||
|
||||
# unused kivy files (platform specific)
|
||||
kivy/input/providers/wm_*
|
||||
kivy/input/providers/mactouch*
|
||||
kivy/input/providers/probesysfs*
|
||||
kivy/input/providers/mtdev*
|
||||
kivy/input/providers/hidinput*
|
||||
kivy/core/camera/camera_videocapture*
|
||||
kivy/core/spelling/*osx*
|
||||
kivy/core/video/video_pyglet*
|
||||
kivy/tools
|
||||
kivy/tests/*
|
||||
kivy/*/*.h
|
||||
kivy/*/*.pxi
|
||||
|
||||
# unused encodings
|
||||
lib-dynload/*codec*
|
||||
encodings/cp*.pyo
|
||||
encodings/tis*
|
||||
encodings/shift*
|
||||
encodings/bz2*
|
||||
encodings/iso*
|
||||
encodings/undefined*
|
||||
encodings/johab*
|
||||
encodings/p*
|
||||
encodings/m*
|
||||
encodings/euc*
|
||||
encodings/k*
|
||||
encodings/unicode_internal*
|
||||
encodings/quo*
|
||||
encodings/gb*
|
||||
encodings/big5*
|
||||
encodings/hp*
|
||||
encodings/hz*
|
||||
|
||||
# unused python modules
|
||||
bsddb/*
|
||||
wsgiref/*
|
||||
hotshot/*
|
||||
pydoc_data/*
|
||||
tty.pyo
|
||||
anydbm.pyo
|
||||
nturl2path.pyo
|
||||
LICENCE.txt
|
||||
macurl2path.pyo
|
||||
dummy_threading.pyo
|
||||
audiodev.pyo
|
||||
antigravity.pyo
|
||||
dumbdbm.pyo
|
||||
sndhdr.pyo
|
||||
__phello__.foo.pyo
|
||||
sunaudio.pyo
|
||||
os2emxpath.pyo
|
||||
multiprocessing/dummy*
|
||||
|
||||
# unused binaries python modules
|
||||
lib-dynload/termios.so
|
||||
lib-dynload/_lsprof.so
|
||||
lib-dynload/*audioop.so
|
||||
lib-dynload/_hotshot.so
|
||||
lib-dynload/_heapq.so
|
||||
lib-dynload/_json.so
|
||||
lib-dynload/grp.so
|
||||
lib-dynload/resource.so
|
||||
lib-dynload/pyexpat.so
|
||||
lib-dynload/_ctypes_test.so
|
||||
lib-dynload/_testcapi.so
|
||||
|
||||
# odd files
|
||||
plat-linux3/regen
|
||||
|
||||
#>sqlite3
|
||||
# conditionnal include depending if some recipes are included or not.
|
||||
sqlite3/*
|
||||
lib-dynload/_sqlite3.so
|
||||
#<sqlite3
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
include $(call all-subdir-makefiles)
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
# Uncomment this if you're using STL in your project
|
||||
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
|
||||
# APP_STL := stlport_static
|
||||
|
||||
# APP_ABI := armeabi armeabi-v7a x86
|
||||
APP_ABI := $(ARCH)
|
||||
@@ -0,0 +1,22 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
# Add your application source files here...
|
||||
LOCAL_SRC_FILES := start.c pyjniusjni.c
|
||||
|
||||
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
|
||||
|
||||
LOCAL_SHARED_LIBRARIES := python_shared
|
||||
|
||||
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
|
||||
|
||||
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
|
||||
ifdef CRYSTAX_PYTHON_VERSION
|
||||
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
|
||||
endif
|
||||
@@ -0,0 +1,10 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
LOCAL_SRC_FILES := YourSourceHere.c
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
#define BOOTSTRAP_NAME_SERVICEONLY
|
||||
#define BOOTSTRAP_USES_NO_SDL_HEADERS
|
||||
|
||||
const char bootstrap_name[] = "service_only";
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
#include <pthread.h>
|
||||
#include <jni.h>
|
||||
|
||||
#define LOGI(...) do {} while (0)
|
||||
#define LOGE(...) do {} while (0)
|
||||
|
||||
#include "android/log.h"
|
||||
|
||||
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
|
||||
|
||||
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
|
||||
/* #define LOGP(x) LOG("python", (x)) */
|
||||
#define LOG_TAG "Python_android"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
|
||||
|
||||
|
||||
/* Function headers */
|
||||
JNIEnv* Android_JNI_GetEnv(void);
|
||||
static void Android_JNI_ThreadDestroyed(void*);
|
||||
|
||||
static pthread_key_t mThreadKey;
|
||||
static JavaVM* mJavaVM;
|
||||
|
||||
int Android_JNI_SetupThread(void)
|
||||
{
|
||||
Android_JNI_GetEnv();
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Library init */
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
|
||||
{
|
||||
JNIEnv *env;
|
||||
mJavaVM = vm;
|
||||
LOGI("JNI_OnLoad called");
|
||||
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
|
||||
LOGE("Failed to get the environment using GetEnv()");
|
||||
return -1;
|
||||
}
|
||||
/*
|
||||
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
|
||||
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
|
||||
*/
|
||||
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
|
||||
|
||||
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
|
||||
}
|
||||
Android_JNI_SetupThread();
|
||||
|
||||
return JNI_VERSION_1_4;
|
||||
}
|
||||
|
||||
JNIEnv* Android_JNI_GetEnv(void)
|
||||
{
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* All threads are Linux threads, scheduled by the kernel.
|
||||
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
|
||||
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
|
||||
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
|
||||
* and cannot make JNI calls.
|
||||
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
|
||||
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
|
||||
* is a no-op.
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
*/
|
||||
|
||||
JNIEnv *env;
|
||||
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
|
||||
if(status < 0) {
|
||||
LOGE("failed to attach current thread");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
|
||||
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
|
||||
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
|
||||
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
|
||||
* Note: The destructor is not called unless the stored value is != NULL
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
* (except for some lost CPU cycles)
|
||||
*/
|
||||
pthread_setspecific(mThreadKey, (void*) env);
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
static void Android_JNI_ThreadDestroyed(void* value)
|
||||
{
|
||||
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
|
||||
JNIEnv *env = (JNIEnv*) value;
|
||||
if (env != NULL) {
|
||||
(*mJavaVM)->DetachCurrentThread(mJavaVM);
|
||||
pthread_setspecific(mThreadKey, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
void *WebView_AndroidGetJNIEnv()
|
||||
{
|
||||
return Android_JNI_GetEnv();
|
||||
}
|
||||
Whitespace-only changes.
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class Octal {
|
||||
|
||||
/**
|
||||
* Parse an octal string from a header buffer. This is used for the file
|
||||
* permission mode value.
|
||||
*
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal string.
|
||||
*/
|
||||
public static long parseOctal(byte[] header, int offset, int length) {
|
||||
long result = 0;
|
||||
boolean stillPadding = true;
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
|
||||
if (header[i] == (byte) ' ' || header[i] == '0') {
|
||||
if (stillPadding)
|
||||
continue;
|
||||
|
||||
if (header[i] == (byte) ' ')
|
||||
break;
|
||||
}
|
||||
|
||||
stillPadding = false;
|
||||
|
||||
result = ( result << 3 ) + ( header[i] - '0' );
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The integer value of the octal bytes.
|
||||
*/
|
||||
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
int idx = length - 1;
|
||||
|
||||
buf[offset + idx] = 0;
|
||||
--idx;
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
--idx;
|
||||
|
||||
if (value == 0) {
|
||||
buf[offset + idx] = (byte) '0';
|
||||
--idx;
|
||||
} else {
|
||||
for (long val = value; idx >= 0 && val > 0; --idx) {
|
||||
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
|
||||
val = val >> 3;
|
||||
}
|
||||
}
|
||||
|
||||
for (; idx >= 0; --idx) {
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the checksum octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The integer value of the entry's checksum.
|
||||
*/
|
||||
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
getOctalBytes( value, buf, offset, length );
|
||||
buf[offset + length - 1] = (byte) ' ';
|
||||
buf[offset + length - 2] = 0;
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal long integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal bytes.
|
||||
*/
|
||||
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
byte[] temp = new byte[length + 1];
|
||||
getOctalBytes( value, temp, 0, length + 1 );
|
||||
System.arraycopy( temp, 0, buf, offset, length );
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarConstants {
|
||||
public static final int EOF_BLOCK = 1024;
|
||||
public static final int DATA_BLOCK = 512;
|
||||
public static final int HEADER_BLOCK = 512;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarEntry {
|
||||
protected File file;
|
||||
protected TarHeader header;
|
||||
|
||||
private TarEntry() {
|
||||
this.file = null;
|
||||
header = new TarHeader();
|
||||
}
|
||||
|
||||
public TarEntry(File file, String entryName) {
|
||||
this();
|
||||
this.file = file;
|
||||
this.extractTarHeader(entryName);
|
||||
}
|
||||
|
||||
public TarEntry(byte[] headerBuf) {
|
||||
this();
|
||||
this.parseTarHeader(headerBuf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor to create an entry from an existing TarHeader object.
|
||||
*
|
||||
* This method is useful to add new entries programmatically (e.g. for
|
||||
* adding files or directories that do not exist in the file system).
|
||||
*
|
||||
* @param header
|
||||
*
|
||||
*/
|
||||
public TarEntry(TarHeader header) {
|
||||
this.file = null;
|
||||
this.header = header;
|
||||
}
|
||||
|
||||
public boolean equals(TarEntry it) {
|
||||
return header.name.toString().equals(it.header.name.toString());
|
||||
}
|
||||
|
||||
public boolean isDescendent(TarEntry desc) {
|
||||
return desc.header.name.toString().startsWith(header.name.toString());
|
||||
}
|
||||
|
||||
public TarHeader getHeader() {
|
||||
return header;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
String name = header.name.toString();
|
||||
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
|
||||
name = header.namePrefix.toString() + "/" + name;
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
public int getUserId() {
|
||||
return header.userId;
|
||||
}
|
||||
|
||||
public void setUserId(int userId) {
|
||||
header.userId = userId;
|
||||
}
|
||||
|
||||
public int getGroupId() {
|
||||
return header.groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(int groupId) {
|
||||
header.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return header.userName.toString();
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
header.userName = new StringBuffer(userName);
|
||||
}
|
||||
|
||||
public String getGroupName() {
|
||||
return header.groupName.toString();
|
||||
}
|
||||
|
||||
public void setGroupName(String groupName) {
|
||||
header.groupName = new StringBuffer(groupName);
|
||||
}
|
||||
|
||||
public void setIds(int userId, int groupId) {
|
||||
this.setUserId(userId);
|
||||
this.setGroupId(groupId);
|
||||
}
|
||||
|
||||
public void setModTime(long time) {
|
||||
header.modTime = time / 1000;
|
||||
}
|
||||
|
||||
public void setModTime(Date time) {
|
||||
header.modTime = time.getTime() / 1000;
|
||||
}
|
||||
|
||||
public Date getModTime() {
|
||||
return new Date(header.modTime * 1000);
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return this.file;
|
||||
}
|
||||
|
||||
public long getSize() {
|
||||
return header.size;
|
||||
}
|
||||
|
||||
public void setSize(long size) {
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the org.kamrazafar.jtar entry is a directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean isDirectory() {
|
||||
if (this.file != null)
|
||||
return this.file.isDirectory();
|
||||
|
||||
if (header != null) {
|
||||
if (header.linkFlag == TarHeader.LF_DIR)
|
||||
return true;
|
||||
|
||||
if (header.name.toString().endsWith("/"))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract header from File
|
||||
*
|
||||
* @param entryName
|
||||
*/
|
||||
public void extractTarHeader(String entryName) {
|
||||
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum
|
||||
*
|
||||
* @param buf
|
||||
* @return
|
||||
*/
|
||||
public long computeCheckSum(byte[] buf) {
|
||||
long sum = 0;
|
||||
|
||||
for (int i = 0; i < buf.length; ++i) {
|
||||
sum += 255 & buf[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the header to the byte buffer
|
||||
*
|
||||
* @param outbuf
|
||||
*/
|
||||
public void writeEntryHeader(byte[] outbuf) {
|
||||
int offset = 0;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
|
||||
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
|
||||
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
|
||||
|
||||
long size = header.size;
|
||||
|
||||
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
|
||||
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
|
||||
|
||||
int csOffset = offset;
|
||||
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
|
||||
outbuf[offset++] = (byte) ' ';
|
||||
|
||||
outbuf[offset++] = header.linkFlag;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
|
||||
for (; offset < outbuf.length;)
|
||||
outbuf[offset++] = 0;
|
||||
|
||||
long checkSum = this.computeCheckSum(outbuf);
|
||||
|
||||
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the tar header to the byte buffer
|
||||
*
|
||||
* @param header
|
||||
* @param bh
|
||||
*/
|
||||
public void parseTarHeader(byte[] bh) {
|
||||
int offset = 0;
|
||||
|
||||
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
|
||||
offset += TarHeader.MODELEN;
|
||||
|
||||
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
|
||||
offset += TarHeader.UIDLEN;
|
||||
|
||||
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
|
||||
offset += TarHeader.GIDLEN;
|
||||
|
||||
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
|
||||
offset += TarHeader.SIZELEN;
|
||||
|
||||
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
|
||||
offset += TarHeader.MODTIMELEN;
|
||||
|
||||
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
|
||||
offset += TarHeader.CHKSUMLEN;
|
||||
|
||||
header.linkFlag = bh[offset++];
|
||||
|
||||
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset += TarHeader.USTAR_MAGICLEN;
|
||||
|
||||
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset += TarHeader.USTAR_USER_NAMELEN;
|
||||
|
||||
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset += TarHeader.USTAR_GROUP_NAMELEN;
|
||||
|
||||
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 0 100 File name
|
||||
* 100 8 File mode
|
||||
* 108 8 Owner's numeric user ID
|
||||
* 116 8 Group's numeric user ID
|
||||
* 124 12 File size in bytes
|
||||
* 136 12 Last modification time in numeric Unix time format
|
||||
* 148 8 Checksum for header block
|
||||
* 156 1 Link indicator (file type)
|
||||
* 157 100 Name of linked file
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* File Types
|
||||
*
|
||||
* <pre>
|
||||
* Value Meaning
|
||||
* '0' Normal file
|
||||
* (ASCII NUL) Normal file (now obsolete)
|
||||
* '1' Hard link
|
||||
* '2' Symbolic link
|
||||
* '3' Character special
|
||||
* '4' Block special
|
||||
* '5' Directory
|
||||
* '6' FIFO
|
||||
* '7' Contigous
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
*
|
||||
* Ustar header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 257 6 UStar indicator "ustar"
|
||||
* 263 2 UStar version "00"
|
||||
* 265 32 Owner user name
|
||||
* 297 32 Owner group name
|
||||
* 329 8 Device major number
|
||||
* 337 8 Device minor number
|
||||
* 345 155 Filename prefix
|
||||
* </pre>
|
||||
*/
|
||||
|
||||
public class TarHeader {
|
||||
|
||||
/*
|
||||
* Header
|
||||
*/
|
||||
public static final int NAMELEN = 100;
|
||||
public static final int MODELEN = 8;
|
||||
public static final int UIDLEN = 8;
|
||||
public static final int GIDLEN = 8;
|
||||
public static final int SIZELEN = 12;
|
||||
public static final int MODTIMELEN = 12;
|
||||
public static final int CHKSUMLEN = 8;
|
||||
public static final byte LF_OLDNORM = 0;
|
||||
|
||||
/*
|
||||
* File Types
|
||||
*/
|
||||
public static final byte LF_NORMAL = (byte) '0';
|
||||
public static final byte LF_LINK = (byte) '1';
|
||||
public static final byte LF_SYMLINK = (byte) '2';
|
||||
public static final byte LF_CHR = (byte) '3';
|
||||
public static final byte LF_BLK = (byte) '4';
|
||||
public static final byte LF_DIR = (byte) '5';
|
||||
public static final byte LF_FIFO = (byte) '6';
|
||||
public static final byte LF_CONTIG = (byte) '7';
|
||||
|
||||
/*
|
||||
* Ustar header
|
||||
*/
|
||||
|
||||
public static final String USTAR_MAGIC = "ustar"; // POSIX
|
||||
|
||||
public static final int USTAR_MAGICLEN = 8;
|
||||
public static final int USTAR_USER_NAMELEN = 32;
|
||||
public static final int USTAR_GROUP_NAMELEN = 32;
|
||||
public static final int USTAR_DEVLEN = 8;
|
||||
public static final int USTAR_FILENAME_PREFIX = 155;
|
||||
|
||||
// Header values
|
||||
public StringBuffer name;
|
||||
public int mode;
|
||||
public int userId;
|
||||
public int groupId;
|
||||
public long size;
|
||||
public long modTime;
|
||||
public int checkSum;
|
||||
public byte linkFlag;
|
||||
public StringBuffer linkName;
|
||||
public StringBuffer magic; // ustar indicator and version
|
||||
public StringBuffer userName;
|
||||
public StringBuffer groupName;
|
||||
public int devMajor;
|
||||
public int devMinor;
|
||||
public StringBuffer namePrefix;
|
||||
|
||||
public TarHeader() {
|
||||
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
|
||||
|
||||
this.name = new StringBuffer();
|
||||
this.linkName = new StringBuffer();
|
||||
|
||||
String user = System.getProperty("user.name", "");
|
||||
|
||||
if (user.length() > 31)
|
||||
user = user.substring(0, 31);
|
||||
|
||||
this.userId = 0;
|
||||
this.groupId = 0;
|
||||
this.userName = new StringBuffer(user);
|
||||
this.groupName = new StringBuffer("");
|
||||
this.namePrefix = new StringBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an entry name from a header buffer.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The header's entry name.
|
||||
*/
|
||||
public static StringBuffer parseName(byte[] header, int offset, int length) {
|
||||
StringBuffer result = new StringBuffer(length);
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
result.append((char) header[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the number of bytes in an entry name.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The number of bytes in a header's entry name.
|
||||
*/
|
||||
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < length && i < name.length(); ++i) {
|
||||
buf[offset + i] = (byte) name.charAt(i);
|
||||
}
|
||||
|
||||
for (; i < length; ++i) {
|
||||
buf[offset + i] = 0;
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new header for a file/directory entry.
|
||||
*
|
||||
*
|
||||
* @param name
|
||||
* File name
|
||||
* @param size
|
||||
* File size in bytes
|
||||
* @param modTime
|
||||
* Last modification time in numeric Unix time format
|
||||
* @param dir
|
||||
* Is directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
|
||||
String name = entryName;
|
||||
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
|
||||
|
||||
TarHeader header = new TarHeader();
|
||||
header.linkName = new StringBuffer("");
|
||||
|
||||
if (name.length() > 100) {
|
||||
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
|
||||
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
|
||||
} else {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
if (dir) {
|
||||
header.mode = 040755;
|
||||
header.linkFlag = TarHeader.LF_DIR;
|
||||
if (header.name.charAt(header.name.length() - 1) != '/') {
|
||||
header.name.append("/");
|
||||
}
|
||||
header.size = 0;
|
||||
} else {
|
||||
header.mode = 0100644;
|
||||
header.linkFlag = TarHeader.LF_NORMAL;
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
header.modTime = modTime;
|
||||
header.checkSum = 0;
|
||||
header.devMajor = 0;
|
||||
header.devMinor = 0;
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
+249
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarInputStream extends FilterInputStream {
|
||||
|
||||
private static final int SKIP_BUFFER_SIZE = 2048;
|
||||
private TarEntry currentEntry;
|
||||
private long currentFileSize;
|
||||
private long bytesRead;
|
||||
private boolean defaultSkip = false;
|
||||
|
||||
public TarInputStream(InputStream in) {
|
||||
super(in);
|
||||
currentFileSize = 0;
|
||||
bytesRead = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void mark(int readlimit) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void reset() throws IOException {
|
||||
throw new IOException("mark/reset not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a byte
|
||||
*
|
||||
* @see java.io.FilterInputStream#read()
|
||||
*/
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
byte[] buf = new byte[1];
|
||||
|
||||
int res = this.read(buf, 0, 1);
|
||||
|
||||
if (res != -1) {
|
||||
return 0xFF & buf[0];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being read exceed the entry size and adjusts the byte
|
||||
* array length. Updates the byte counters
|
||||
*
|
||||
*
|
||||
* @see java.io.FilterInputStream#read(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentFileSize == currentEntry.getSize()) {
|
||||
return -1;
|
||||
} else if ((currentEntry.getSize() - currentFileSize) < len) {
|
||||
len = (int) (currentEntry.getSize() - currentFileSize);
|
||||
}
|
||||
}
|
||||
|
||||
int br = super.read(b, off, len);
|
||||
|
||||
if (br != -1) {
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += br;
|
||||
}
|
||||
|
||||
bytesRead += br;
|
||||
}
|
||||
|
||||
return br;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next entry in the tar file
|
||||
*
|
||||
* @return TarEntry
|
||||
* @throws IOException
|
||||
*/
|
||||
public TarEntry getNextEntry() throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
|
||||
int tr = 0;
|
||||
|
||||
// Read full header
|
||||
while (tr < TarConstants.HEADER_BLOCK) {
|
||||
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
|
||||
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
System.arraycopy(theader, 0, header, tr, res);
|
||||
tr += res;
|
||||
}
|
||||
|
||||
// Check if record is null
|
||||
boolean eof = true;
|
||||
for (byte b : header) {
|
||||
if (b != 0) {
|
||||
eof = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!eof) {
|
||||
currentEntry = new TarEntry(header);
|
||||
}
|
||||
|
||||
return currentEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current offset (in bytes) from the beginning of the stream.
|
||||
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
|
||||
*/
|
||||
public long getCurrentOffset() {
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
// Not fully read, skip rest of the bytes
|
||||
long bs = 0;
|
||||
while (bs < currentEntry.getSize() - currentFileSize) {
|
||||
long res = skip(currentEntry.getSize() - currentFileSize - bs);
|
||||
|
||||
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
|
||||
// I suspect file corruption
|
||||
throw new IOException("Possible tar file corruption");
|
||||
}
|
||||
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0L;
|
||||
skipPad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips the pad at the end of each tar entry file content
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void skipPad() throws IOException {
|
||||
if (bytesRead > 0) {
|
||||
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
|
||||
|
||||
if (extra > 0) {
|
||||
long bs = 0;
|
||||
while (bs < TarConstants.DATA_BLOCK - extra) {
|
||||
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips 'n' bytes on the InputStream<br>
|
||||
* Overrides default implementation of skip
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
if (defaultSkip) {
|
||||
// use skip method of parent stream
|
||||
// may not work if skip not implemented by parent
|
||||
long bs = super.skip(n);
|
||||
bytesRead += bs;
|
||||
|
||||
return bs;
|
||||
}
|
||||
|
||||
if (n <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
long left = n;
|
||||
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
|
||||
|
||||
while (left > 0) {
|
||||
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
left -= res;
|
||||
}
|
||||
|
||||
return n - left;
|
||||
}
|
||||
|
||||
public boolean isDefaultSkip() {
|
||||
return defaultSkip;
|
||||
}
|
||||
|
||||
public void setDefaultSkip(boolean defaultSkip) {
|
||||
this.defaultSkip = defaultSkip;
|
||||
}
|
||||
}
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarOutputStream extends OutputStream {
|
||||
private final OutputStream out;
|
||||
private long bytesWritten;
|
||||
private long currentFileSize;
|
||||
private TarEntry currentEntry;
|
||||
|
||||
public TarOutputStream(OutputStream out) {
|
||||
this.out = out;
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
public TarOutputStream(final File fout) throws FileNotFoundException {
|
||||
this.out = new BufferedOutputStream(new FileOutputStream(fout));
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a file for writing.
|
||||
*/
|
||||
public TarOutputStream(final File fout, final boolean append) throws IOException {
|
||||
@SuppressWarnings("resource")
|
||||
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
|
||||
final long fileSize = fout.length();
|
||||
if (append && fileSize > TarConstants.EOF_BLOCK) {
|
||||
raf.seek(fileSize - TarConstants.EOF_BLOCK);
|
||||
}
|
||||
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the EOF record and closes the stream
|
||||
*
|
||||
* @see java.io.FilterOutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closeCurrentEntry();
|
||||
write( new byte[TarConstants.EOF_BLOCK] );
|
||||
out.close();
|
||||
}
|
||||
/**
|
||||
* Writes a byte to the stream and updates byte counters
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
out.write( b );
|
||||
bytesWritten += 1;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being written exceed the current entry size.
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null && !currentEntry.isDirectory()) {
|
||||
if (currentEntry.getSize() < currentFileSize + len) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
|
||||
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
|
||||
+ "] being written." );
|
||||
}
|
||||
}
|
||||
|
||||
out.write( b, off, len );
|
||||
|
||||
bytesWritten += len;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += len;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the next tar entry header on the stream
|
||||
*
|
||||
* @param entry
|
||||
* @throws IOException
|
||||
*/
|
||||
public void putNextEntry(TarEntry entry) throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
entry.writeEntryHeader( header );
|
||||
|
||||
write( header );
|
||||
|
||||
currentEntry = entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
|
||||
+ currentEntry.getSize() + "] has not been fully written." );
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0;
|
||||
|
||||
pad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pads the last content block
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void pad() throws IOException {
|
||||
if (bytesWritten > 0) {
|
||||
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
|
||||
|
||||
if (extra > 0) {
|
||||
write( new byte[TarConstants.DATA_BLOCK - extra] );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author Kamran
|
||||
*
|
||||
*/
|
||||
public class TarUtils {
|
||||
/**
|
||||
* Determines the tar file size of the given folder/file path
|
||||
*
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
public static long calculateTarSize(File path) {
|
||||
return tarSize(path) + TarConstants.EOF_BLOCK;
|
||||
}
|
||||
|
||||
private static long tarSize(File dir) {
|
||||
long size = 0;
|
||||
|
||||
if (dir.isFile()) {
|
||||
return entrySize(dir.length());
|
||||
} else {
|
||||
File[] subFiles = dir.listFiles();
|
||||
|
||||
if (subFiles != null && subFiles.length > 0) {
|
||||
for (File file : subFiles) {
|
||||
if (file.isFile()) {
|
||||
size += entrySize(file.length());
|
||||
} else {
|
||||
size += tarSize(file);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Empty folder header
|
||||
return TarConstants.HEADER_BLOCK;
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
private static long entrySize(long fileSize) {
|
||||
long size = 0;
|
||||
size += TarConstants.HEADER_BLOCK; // Header
|
||||
size += fileSize; // File size
|
||||
|
||||
long extra = size % TarConstants.DATA_BLOCK;
|
||||
|
||||
if (extra > 0) {
|
||||
size += (TarConstants.DATA_BLOCK - extra); // pad
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
public static String trim(String s, char c) {
|
||||
StringBuffer tmp = new StringBuffer(s);
|
||||
for (int i = 0; i < tmp.length(); i++) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = tmp.length() - 1; i >= 0; i--) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
return tmp.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
|
||||
package org.kivy.android;
|
||||
|
||||
import java.net.Socket;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import android.os.SystemClock;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import android.app.*;
|
||||
import android.content.*;
|
||||
import android.view.*;
|
||||
import android.view.SurfaceView;
|
||||
import android.app.Activity;
|
||||
import android.content.Intent;
|
||||
import android.util.Log;
|
||||
import android.widget.Toast;
|
||||
import android.os.AsyncTask;
|
||||
import android.os.Bundle;
|
||||
import android.os.PowerManager;
|
||||
import android.graphics.PixelFormat;
|
||||
import android.view.SurfaceHolder;
|
||||
import android.content.Context;
|
||||
import android.content.pm.ActivityInfo;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.pm.ApplicationInfo;
|
||||
import android.content.Intent;
|
||||
import android.widget.ImageView;
|
||||
import java.io.InputStream;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.Color;
|
||||
|
||||
import android.widget.AbsoluteLayout;
|
||||
|
||||
import android.webkit.WebViewClient;
|
||||
import android.webkit.WebView;
|
||||
|
||||
import org.kivy.android.PythonUtil;
|
||||
|
||||
import org.renpy.android.ResourceManager;
|
||||
import org.renpy.android.AssetExtract;
|
||||
|
||||
public class PythonActivity extends Activity {
|
||||
// This activity is modified from a mixture of the SDLActivity and
|
||||
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
|
||||
// specifics.
|
||||
|
||||
private static final String TAG = "PythonActivity";
|
||||
|
||||
public static PythonActivity mActivity = null;
|
||||
|
||||
/** If shared libraries (e.g. the native application) could not be loaded. */
|
||||
public static boolean mBrokenLibraries;
|
||||
|
||||
protected static Thread mPythonThread;
|
||||
|
||||
private ResourceManager resourceManager = null;
|
||||
private Bundle mMetaData = null;
|
||||
private PowerManager.WakeLock mWakeLock = null;
|
||||
|
||||
public String getAppRoot() {
|
||||
String app_root = getFilesDir().getAbsolutePath() + "/app";
|
||||
return app_root;
|
||||
}
|
||||
|
||||
public static void initialize() {
|
||||
// The static nature of the singleton and Android quirkiness force us to initialize everything here
|
||||
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
|
||||
mBrokenLibraries = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
Log.v(TAG, "My oncreate running");
|
||||
resourceManager = new ResourceManager(this);
|
||||
|
||||
Log.v(TAG, "Ready to unpack");
|
||||
File app_root_file = new File(getAppRoot());
|
||||
unpackData("private", app_root_file);
|
||||
|
||||
Log.v(TAG, "About to do super onCreate");
|
||||
super.onCreate(savedInstanceState);
|
||||
Log.v(TAG, "Did super onCreate");
|
||||
|
||||
this.mActivity = this;
|
||||
//this.showLoadingScreen();
|
||||
Log.v("Python", "Device: " + android.os.Build.DEVICE);
|
||||
Log.v("Python", "Model: " + android.os.Build.MODEL);
|
||||
|
||||
//Log.v(TAG, "Ready to unpack");
|
||||
//new UnpackFilesTask().execute(getAppRoot());
|
||||
|
||||
PythonActivity.initialize();
|
||||
|
||||
// Load shared libraries
|
||||
String errorMsgBrokenLib = "";
|
||||
try {
|
||||
loadLibraries();
|
||||
} catch(UnsatisfiedLinkError e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
} catch(Exception e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
}
|
||||
|
||||
if (mBrokenLibraries)
|
||||
{
|
||||
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
|
||||
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
|
||||
+ System.getProperty("line.separator")
|
||||
+ System.getProperty("line.separator")
|
||||
+ "Error: " + errorMsgBrokenLib);
|
||||
dlgAlert.setTitle("Python Error");
|
||||
dlgAlert.setPositiveButton("Exit",
|
||||
new DialogInterface.OnClickListener() {
|
||||
@Override
|
||||
public void onClick(DialogInterface dialog,int id) {
|
||||
// if this button is clicked, close current activity
|
||||
PythonActivity.mActivity.finish();
|
||||
}
|
||||
});
|
||||
dlgAlert.setCancelable(false);
|
||||
dlgAlert.create().show();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Set up the Python environment
|
||||
String app_root_dir = getAppRoot();
|
||||
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
|
||||
|
||||
Log.v(TAG, "Setting env vars for start.c and Python to use");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
|
||||
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
|
||||
|
||||
try {
|
||||
Log.v(TAG, "Access to our meta-data...");
|
||||
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
|
||||
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
|
||||
|
||||
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
|
||||
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
|
||||
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
|
||||
mActivity.mWakeLock.acquire();
|
||||
}
|
||||
} catch (PackageManager.NameNotFoundException e) {
|
||||
}
|
||||
|
||||
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
|
||||
PythonActivity.mPythonThread = pythonThread;
|
||||
pythonThread.start();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDestroy() {
|
||||
Log.i("Destroy", "end of app");
|
||||
super.onDestroy();
|
||||
|
||||
// make sure all child threads (python_thread) are stopped
|
||||
android.os.Process.killProcess(android.os.Process.myPid());
|
||||
}
|
||||
|
||||
public void loadLibraries() {
|
||||
String app_root = new String(getAppRoot());
|
||||
File app_root_file = new File(app_root);
|
||||
PythonUtil.loadLibraries(app_root_file);
|
||||
}
|
||||
|
||||
public void recursiveDelete(File f) {
|
||||
if (f.isDirectory()) {
|
||||
for (File r : f.listFiles()) {
|
||||
recursiveDelete(r);
|
||||
}
|
||||
}
|
||||
f.delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Show an error using a toast. (Only makes sense from non-UI
|
||||
* threads.)
|
||||
*/
|
||||
public void toastError(final String msg) {
|
||||
|
||||
final Activity thisActivity = this;
|
||||
|
||||
runOnUiThread(new Runnable () {
|
||||
public void run() {
|
||||
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
|
||||
}
|
||||
});
|
||||
|
||||
// Wait to show the error.
|
||||
synchronized (this) {
|
||||
try {
|
||||
this.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void unpackData(final String resource, File target) {
|
||||
|
||||
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
|
||||
|
||||
// The version of data in memory and on disk.
|
||||
String data_version = resourceManager.getString(resource + "_version");
|
||||
String disk_version = null;
|
||||
|
||||
Log.v(TAG, "Data version is " + data_version);
|
||||
|
||||
// If no version, no unpacking is necessary.
|
||||
if (data_version == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check the current disk version, if any.
|
||||
String filesDir = target.getAbsolutePath();
|
||||
String disk_version_fn = filesDir + "/" + resource + ".version";
|
||||
|
||||
try {
|
||||
byte buf[] = new byte[64];
|
||||
InputStream is = new FileInputStream(disk_version_fn);
|
||||
int len = is.read(buf);
|
||||
disk_version = new String(buf, 0, len);
|
||||
is.close();
|
||||
} catch (Exception e) {
|
||||
disk_version = "";
|
||||
}
|
||||
|
||||
// If the disk data is out of date, extract it and write the
|
||||
// version file.
|
||||
// if (! data_version.equals(disk_version)) {
|
||||
if (! data_version.equals(disk_version)) {
|
||||
Log.v(TAG, "Extracting " + resource + " assets.");
|
||||
|
||||
recursiveDelete(target);
|
||||
target.mkdirs();
|
||||
|
||||
AssetExtract ae = new AssetExtract(this);
|
||||
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
|
||||
toastError("Could not extract " + resource + " data.");
|
||||
}
|
||||
|
||||
try {
|
||||
// Write .nomedia.
|
||||
new File(target, ".nomedia").createNewFile();
|
||||
|
||||
// Write version file.
|
||||
FileOutputStream os = new FileOutputStream(disk_version_fn);
|
||||
os.write(data_version.getBytes());
|
||||
os.close();
|
||||
} catch (Exception e) {
|
||||
Log.w("python", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long lastBackClick = SystemClock.elapsedRealtime();
|
||||
@Override
|
||||
public boolean onKeyDown(int keyCode, KeyEvent event) {
|
||||
// If it wasn't the Back key or there's no web page history, bubble up to the default
|
||||
// system behavior (probably exit the activity)
|
||||
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
Toast.makeText(this, "Click again to close the app",
|
||||
Toast.LENGTH_LONG).show();
|
||||
return true;
|
||||
}
|
||||
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
return super.onKeyDown(keyCode, event);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onNewIntent
|
||||
//
|
||||
|
||||
public interface NewIntentListener {
|
||||
void onNewIntent(Intent intent);
|
||||
}
|
||||
|
||||
private List<NewIntentListener> newIntentListeners = null;
|
||||
|
||||
public void registerNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
|
||||
this.newIntentListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.newIntentListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onNewIntent(Intent intent) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.newIntentListeners ) {
|
||||
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
|
||||
while ( iterator.hasNext() ) {
|
||||
(iterator.next()).onNewIntent(intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onActivityResult
|
||||
//
|
||||
|
||||
public interface ActivityResultListener {
|
||||
void onActivityResult(int requestCode, int resultCode, Intent data);
|
||||
}
|
||||
|
||||
private List<ActivityResultListener> activityResultListeners = null;
|
||||
|
||||
public void registerActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
|
||||
this.activityResultListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.activityResultListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.activityResultListeners ) {
|
||||
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
|
||||
while ( iterator.hasNext() )
|
||||
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
|
||||
}
|
||||
}
|
||||
|
||||
public static void start_service(String serviceTitle, String serviceDescription,
|
||||
String pythonServiceArgument) {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
|
||||
String filesDirectory = argument;
|
||||
String app_root_dir = PythonActivity.mActivity.getAppRoot();
|
||||
serviceIntent.putExtra("androidPrivate", argument);
|
||||
serviceIntent.putExtra("androidArgument", app_root_dir);
|
||||
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
|
||||
serviceIntent.putExtra("pythonName", "python");
|
||||
serviceIntent.putExtra("pythonHome", app_root_dir);
|
||||
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
serviceIntent.putExtra("serviceTitle", serviceTitle);
|
||||
serviceIntent.putExtra("serviceDescription", serviceDescription);
|
||||
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
PythonActivity.mActivity.startService(serviceIntent);
|
||||
}
|
||||
|
||||
public static void stop_service() {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
PythonActivity.mActivity.stopService(serviceIntent);
|
||||
}
|
||||
|
||||
|
||||
public static native void nativeSetEnv(String j_name, String j_value);
|
||||
public static native int nativeInit(Object arguments);
|
||||
|
||||
}
|
||||
|
||||
|
||||
class PythonMain implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
PythonActivity.nativeInit(new String[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonEvent {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
private final Condition cond = lock.newCondition();
|
||||
private boolean flag = false;
|
||||
|
||||
public void set() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = true;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void wait_() throws InterruptedException {
|
||||
lock.lock();
|
||||
try {
|
||||
while (!flag) {
|
||||
cond.await();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = false;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonLock {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
|
||||
public void acquire() {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package org.renpy.android;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.IntentFilter;
|
||||
import android.hardware.Sensor;
|
||||
import android.hardware.SensorEvent;
|
||||
import android.hardware.SensorEventListener;
|
||||
import android.hardware.SensorManager;
|
||||
import android.net.ConnectivityManager;
|
||||
import android.net.NetworkInfo;
|
||||
import android.net.wifi.ScanResult;
|
||||
import android.net.wifi.WifiManager;
|
||||
import android.os.Vibrator;
|
||||
import android.view.View;
|
||||
|
||||
/**
|
||||
* Methods that are expected to be called via JNI, to access the device's
|
||||
* non-screen hardware. (For example, the vibration and accelerometer.)
|
||||
*/
|
||||
public class Hardware {
|
||||
|
||||
// The context.
|
||||
static Context context;
|
||||
static View view;
|
||||
|
||||
/**
|
||||
* Vibrate for s seconds.
|
||||
*/
|
||||
public static void vibrate(double s) {
|
||||
Vibrator v = (Vibrator) context
|
||||
.getSystemService(Context.VIBRATOR_SERVICE);
|
||||
if (v != null) {
|
||||
v.vibrate((int) (1000 * s));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an Overview of all Hardware Sensors of an Android Device
|
||||
*/
|
||||
public static String getHardwareSensors() {
|
||||
SensorManager sm = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
|
||||
|
||||
if (allSensors != null) {
|
||||
String resultString = "";
|
||||
for (Sensor s : allSensors) {
|
||||
resultString += String.format("Name=" + s.getName());
|
||||
resultString += String.format(",Vendor=" + s.getVendor());
|
||||
resultString += String.format(",Version=" + s.getVersion());
|
||||
resultString += String.format(",MaximumRange="
|
||||
+ s.getMaximumRange());
|
||||
// XXX MinDelay is not in the 2.2
|
||||
// resultString += String.format(",MinDelay=" +
|
||||
// s.getMinDelay());
|
||||
resultString += String.format(",Power=" + s.getPower());
|
||||
resultString += String.format(",Type=" + s.getType() + "\n");
|
||||
}
|
||||
return resultString;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
|
||||
* Magnetic Field Sensors
|
||||
*/
|
||||
public static class generic3AxisSensor implements SensorEventListener {
|
||||
private final SensorManager sSensorManager;
|
||||
private final Sensor sSensor;
|
||||
private final int sSensorType;
|
||||
SensorEvent sSensorEvent;
|
||||
|
||||
public generic3AxisSensor(int sensorType) {
|
||||
sSensorType = sensorType;
|
||||
sSensorManager = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
sSensor = sSensorManager.getDefaultSensor(sSensorType);
|
||||
}
|
||||
|
||||
public void onAccuracyChanged(Sensor sensor, int accuracy) {
|
||||
}
|
||||
|
||||
public void onSensorChanged(SensorEvent event) {
|
||||
sSensorEvent = event;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable the Sensor by registering/unregistering
|
||||
*/
|
||||
public void changeStatus(boolean enable) {
|
||||
if (enable) {
|
||||
sSensorManager.registerListener(this, sSensor,
|
||||
SensorManager.SENSOR_DELAY_NORMAL);
|
||||
} else {
|
||||
sSensorManager.unregisterListener(this, sSensor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the Sensor
|
||||
*/
|
||||
public float[] readSensor() {
|
||||
if (sSensorEvent != null) {
|
||||
return sSensorEvent.values;
|
||||
} else {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static generic3AxisSensor accelerometerSensor = null;
|
||||
public static generic3AxisSensor orientationSensor = null;
|
||||
public static generic3AxisSensor magneticFieldSensor = null;
|
||||
|
||||
/**
|
||||
* functions for backward compatibility reasons
|
||||
*/
|
||||
|
||||
public static void accelerometerEnable(boolean enable) {
|
||||
if (accelerometerSensor == null)
|
||||
accelerometerSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_ACCELEROMETER);
|
||||
accelerometerSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] accelerometerReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (accelerometerSensor == null)
|
||||
return rv;
|
||||
return (float[]) accelerometerSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void orientationSensorEnable(boolean enable) {
|
||||
if (orientationSensor == null)
|
||||
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
|
||||
orientationSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] orientationSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (orientationSensor == null)
|
||||
return rv;
|
||||
return (float[]) orientationSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void magneticFieldSensorEnable(boolean enable) {
|
||||
if (magneticFieldSensor == null)
|
||||
magneticFieldSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_MAGNETIC_FIELD);
|
||||
magneticFieldSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] magneticFieldSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (magneticFieldSensor == null)
|
||||
return rv;
|
||||
return (float[]) magneticFieldSensor.readSensor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan WiFi networks
|
||||
*/
|
||||
static List<ScanResult> latestResult;
|
||||
|
||||
public static void enableWifiScanner() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
|
||||
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
|
||||
// occurs
|
||||
WifiManager w = (WifiManager) c
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
latestResult = w.getScanResults(); // Returns a <list> of
|
||||
// scanResults
|
||||
}
|
||||
|
||||
}, i);
|
||||
|
||||
}
|
||||
|
||||
public static String scanWifi() {
|
||||
|
||||
// Now you can call this and it should execute the broadcastReceiver's
|
||||
// onReceive()
|
||||
WifiManager wm = (WifiManager) context
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
boolean a = wm.startScan();
|
||||
|
||||
if (latestResult != null) {
|
||||
|
||||
String latestResultString = "";
|
||||
for (ScanResult result : latestResult) {
|
||||
latestResultString += String.format("%s\t%s\t%d\n",
|
||||
result.SSID, result.BSSID, result.level);
|
||||
}
|
||||
|
||||
return latestResultString;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* network state
|
||||
*/
|
||||
|
||||
public static boolean network_state = false;
|
||||
|
||||
/**
|
||||
* Check network state directly
|
||||
*
|
||||
* (only one connection can be active at a given moment, detects all network
|
||||
* type)
|
||||
*
|
||||
*/
|
||||
public static boolean checkNetwork() {
|
||||
boolean state = false;
|
||||
final ConnectivityManager conMgr = (ConnectivityManager) context
|
||||
.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||
|
||||
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
|
||||
if (activeNetwork != null && activeNetwork.isConnected()) {
|
||||
state = true;
|
||||
} else {
|
||||
state = false;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* To recieve network state changes
|
||||
*/
|
||||
public static void registerNetworkCheck() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
network_state = checkNetwork();
|
||||
}
|
||||
|
||||
}, i);
|
||||
}
|
||||
|
||||
}
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,93 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="{{ args.package }}"
|
||||
android:versionCode="{{ args.numeric_version }}"
|
||||
android:versionName="{{ args.version }}"
|
||||
android:installLocation="auto">
|
||||
|
||||
<supports-screens
|
||||
android:smallScreens="true"
|
||||
android:normalScreens="true"
|
||||
android:largeScreens="true"
|
||||
android:anyDensity="true"
|
||||
{% if args.min_sdk_version >= 9 %}
|
||||
android:xlargeScreens="true"
|
||||
{% endif %}
|
||||
/>
|
||||
|
||||
<!-- Android 2.3.3 -->
|
||||
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
|
||||
|
||||
<!-- Set permissions -->
|
||||
{% for perm in args.permissions %}
|
||||
{% if '.' in perm %}
|
||||
<uses-permission android:name="{{ perm }}" />
|
||||
{% else %}
|
||||
<uses-permission android:name="android.permission.{{ perm }}" />
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% if args.wakelock %}
|
||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||
{% endif %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<uses-permission android:name="com.android.vending.BILLING" />
|
||||
{% endif %}
|
||||
|
||||
<!-- Create a Java class extending SDLActivity and place it in a
|
||||
directory under src matching the package, e.g.
|
||||
src/com/gamemaker/game/MyGame.java
|
||||
|
||||
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
|
||||
in the XML below.
|
||||
|
||||
An example Java class can be found in README-android.txt
|
||||
-->
|
||||
<application android:label="@string/app_name"
|
||||
android:icon="@drawable/icon"
|
||||
android:allowBackup="true"
|
||||
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
|
||||
android:hardwareAccelerated="true" >
|
||||
|
||||
{% for m in args.meta_data %}
|
||||
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
|
||||
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
|
||||
|
||||
<activity android:name="org.kivy.android.PythonActivity"
|
||||
android:label="@string/app_name"
|
||||
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
|
||||
>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
{%- if args.intent_filters -%}
|
||||
{{- args.intent_filters -}}
|
||||
{%- endif -%}
|
||||
</activity>
|
||||
|
||||
{% if service %}
|
||||
<service android:name="org.kivy.android.PythonService"
|
||||
android:process=":pythonservice" />
|
||||
{% endif %}
|
||||
{% for name in service_names %}
|
||||
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
|
||||
android:process=":service_{{ name }}" />
|
||||
{% endfor %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<service android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbilling" />
|
||||
<receiver android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbillingreceiver">
|
||||
<intent-filter>
|
||||
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
|
||||
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
|
||||
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
{% endif %}
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,74 @@
|
||||
package {{ args.package }};
|
||||
|
||||
import android.os.Binder;
|
||||
import android.os.IBinder;
|
||||
import android.content.Intent;
|
||||
import android.content.Context;
|
||||
import org.kivy.android.PythonService;
|
||||
|
||||
public class Service{{ name|capitalize }} extends PythonService {
|
||||
/**
|
||||
* Binder given to clients
|
||||
*/
|
||||
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
|
||||
|
||||
{% if sticky %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public int getStartType() {
|
||||
return START_STICKY;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% if foreground %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean getStartForeground() {
|
||||
return true;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
public static void start(Context ctx, String pythonServiceArgument) {
|
||||
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
intent.putExtra("androidPrivate", argument);
|
||||
intent.putExtra("androidArgument", argument);
|
||||
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
|
||||
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
|
||||
intent.putExtra("serviceDescription", "");
|
||||
intent.putExtra("pythonName", "{{ name }}");
|
||||
intent.putExtra("pythonHome", argument);
|
||||
intent.putExtra("androidUnpack", argument);
|
||||
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
|
||||
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
ctx.startService(intent);
|
||||
}
|
||||
|
||||
public static void stop(Context ctx) {
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
ctx.stopService(intent);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public IBinder onBind(Intent intent) {
|
||||
return mBinder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Class used for the client Binder. Because we know this service always
|
||||
* runs in the same process as its clients, we don't need to deal with IPC.
|
||||
*/
|
||||
public class Service{{ name|capitalize }}Binder extends Binder {
|
||||
Service{{ name|capitalize }} getService() {
|
||||
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
|
||||
return Service{{ name|capitalize }}.this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">{{ args.name }}</string>
|
||||
<string name="private_version">{{ private_version }}</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,68 @@
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from jnius import PythonJavaClass, autoclass, java_method
|
||||
JAVA_NAMESPACE = 'org.kivy.android'
|
||||
JNI_NAMESPACE = 'org/kivy/android'
|
||||
|
||||
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
|
||||
|
||||
_callbacks = {
|
||||
'on_new_intent': [],
|
||||
'on_activity_result': [],
|
||||
}
|
||||
|
||||
|
||||
class NewIntentListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback, **kwargs):
|
||||
super(NewIntentListener, self).__init__(**kwargs)
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(Landroid/content/Intent;)V')
|
||||
def onNewIntent(self, intent):
|
||||
self.callback(intent)
|
||||
|
||||
|
||||
class ActivityResultListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback):
|
||||
super(ActivityResultListener, self).__init__()
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(IILandroid/content/Intent;)V')
|
||||
def onActivityResult(self, requestCode, resultCode, intent):
|
||||
self.callback(requestCode, resultCode, intent)
|
||||
|
||||
|
||||
def bind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
elif event == 'on_new_intent':
|
||||
listener = NewIntentListener(callback)
|
||||
_activity.registerNewIntentListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
elif event == 'on_activity_result':
|
||||
listener = ActivityResultListener(callback)
|
||||
_activity.registerActivityResultListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
|
||||
|
||||
def unbind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
else:
|
||||
for listener in _callbacks[event][:]:
|
||||
if listener.callback == callback:
|
||||
_callbacks[event].remove(listener)
|
||||
if event == 'on_new_intent':
|
||||
_activity.unregisterNewIntentListener(listener)
|
||||
elif event == 'on_activity_result':
|
||||
_activity.unregisterActivityResultListener(listener)
|
||||
|
||||
|
||||
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
|
||||
@@ -0,0 +1,49 @@
|
||||
from os.path import join, dirname, abspath
|
||||
from pythonforandroid.toolchain import Bootstrap
|
||||
from setuptools import setup, find_packages
|
||||
from bootstrap import LBRYServiceBootstrap
|
||||
|
||||
|
||||
Bootstrap.bootstraps = {
|
||||
'lbry-service': LBRYServiceBootstrap()
|
||||
}
|
||||
|
||||
|
||||
setup(
|
||||
name='lbryservice',
|
||||
version='0.1',
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
description="Android Service for LBRY Network.",
|
||||
license='MIT',
|
||||
python_requires='>=3.7',
|
||||
packages=find_packages(),
|
||||
package_data={'service': ['*.py']},
|
||||
options={
|
||||
'apk': {
|
||||
'dist_name': 'lbry-service',
|
||||
'bootstrap': 'lbry-service',
|
||||
'package': 'io.lbry.service',
|
||||
'permissions': ['INTERNET'],
|
||||
'requirements': ','.join([
|
||||
# needed by aiohttp
|
||||
'multidict', 'yarl', 'async_timeout', 'chardet',
|
||||
# minimum needed by torba:
|
||||
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
|
||||
abspath(join(dirname(__file__), '..', '..', 'torba')),
|
||||
# minimum needed by lbrynet
|
||||
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
|
||||
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
|
||||
abspath(join(dirname(__file__), '..')),
|
||||
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
|
||||
]),
|
||||
'android-api': '26',
|
||||
'ndk-api': '21',
|
||||
'ndk-version': 'r17c',
|
||||
'arch': 'armeabi-v7a',
|
||||
'sdk-dir': '/home/lex/projects/android',
|
||||
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
|
||||
}
|
||||
}
|
||||
)
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.32.0"
|
||||
__version__ = "0.31.0rc3"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+21
-30
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
@@ -18,11 +17,8 @@ from lbrynet.blob.writer import HashBlobWriter
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return len(char) == 1 and _hexmatch.match(char)
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
@@ -33,7 +29,7 @@ def is_valid_blobhash(blobhash: str) -> bool:
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
@@ -70,22 +66,18 @@ class BlobFile:
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if self.file_exists:
|
||||
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
|
||||
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.exception()
|
||||
error = finished.result()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
@@ -94,37 +86,37 @@ class BlobFile:
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
async def save_verified_blob(self, writer):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(verified_bytes)
|
||||
write_handle.write(writer.verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
if os.path.exists(self.file_path):
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
@@ -138,19 +130,18 @@ class BlobFile:
|
||||
"""
|
||||
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
return await self.loop.sendfile(writer.transport, handle)
|
||||
|
||||
def close(self):
|
||||
async def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
self.verified.clear()
|
||||
self.finished_writing.clear()
|
||||
async def delete(self):
|
||||
await self.close()
|
||||
async with self.blob_write_lock:
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlite3 import IntegrityError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -30,21 +30,10 @@ class BlobFileManager:
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
|
||||
raw_blob_hashes = await self.get_all_verified_blobs()
|
||||
self.completed_blob_hashes.update(raw_blob_hashes)
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
@@ -63,28 +52,38 @@ class BlobFileManager:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
|
||||
await self.storage.add_completed_blob(blob.blob_hash)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
async def set_should_announce(self, blob_hash: str, should_announce: bool):
|
||||
now = self.loop.time()
|
||||
return await self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
async def get_all_verified_blobs(self) -> typing.List[str]:
|
||||
blob_hashes = await self.storage.get_all_blob_hashes()
|
||||
return self.check_completed_blobs(blob_hashes)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str]):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash:
|
||||
continue
|
||||
try:
|
||||
blob = self.get_blob(blob_hash)
|
||||
await blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
|
||||
for blob_hash in blob_hashes:
|
||||
self.delete_blob(blob_hash)
|
||||
|
||||
if delete_from_db:
|
||||
await self.storage.delete_blobs_from_db(blob_hashes)
|
||||
if blob_hash in self.blobs:
|
||||
del self.blobs[blob_hash]
|
||||
try:
|
||||
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
@@ -18,6 +18,7 @@ class HashBlobWriter:
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.buffer is not None:
|
||||
@@ -45,7 +46,7 @@ class HashBlobWriter:
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
self.finished.set_result(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
@@ -54,12 +55,15 @@ class HashBlobWriter:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.buffer is not None:
|
||||
|
||||
+81
-117
@@ -1,8 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
@@ -21,184 +19,150 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self.buf = b''
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
response = BlobResponse.deserialize(self.buf + data)
|
||||
if not response.responses and not self._response_fut.done():
|
||||
self.buf += data
|
||||
return
|
||||
else:
|
||||
self.buf = b''
|
||||
response = BlobResponse.deserialize(data)
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
|
||||
|
||||
def _write(self, data):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
def data_received(self, data):
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
self.transport.write(request.serialize())
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
return False, False
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("data rate rejected")
|
||||
return False, False
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.transport
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
return False, True
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("incoming blob hash mismatch")
|
||||
return False, False
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
log.warning("incoming blob unexpected length")
|
||||
return False, False
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
log.info(msg)
|
||||
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
|
||||
return self._blob_bytes_received, self.transport
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
return self._blob_bytes_received, self.close()
|
||||
except (InvalidBlobHashError, InvalidDataError):
|
||||
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
return False, False
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
def close(self):
|
||||
async def close(self):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.exception(e)
|
||||
return self._blob_bytes_received, self.transport
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
self.close()
|
||||
return self._blob_bytes_received, None
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
try:
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or blob.file_exists:
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return 0, None
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
+110
-115
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
@@ -13,125 +13,120 @@ if typing.TYPE_CHECKING:
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
def drain_into(a: list, b: list):
|
||||
while a:
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Set['KademliaPeer'] = set()
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.rounds_won: typing.Dict['KademliaPeer', int] = {}
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'BlobFile'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
|
||||
# the safe net side is that any failure will reset the peer score, triggering the race back
|
||||
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
|
||||
# for peer, task in self.active_connections.items():
|
||||
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
|
||||
# return False
|
||||
return not (blob.get_is_verified() or blob.file_exists)
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
if blob.get_is_verified():
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport
|
||||
)
|
||||
if bytes_received == blob.get_length():
|
||||
self.rounds_won[peer] = self.rounds_won.get(peer, 0) + 1
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored.add(peer)
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.connections[peer] = transport
|
||||
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
|
||||
self.scores[peer] = rough_speed
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
new_peers = await self.peer_queue.get()
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
active_tasks = list(self.active_connections.values())
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
try:
|
||||
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
finally:
|
||||
drain_tasks(tasks)
|
||||
|
||||
def cleanup_active(self):
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(self.peer_queue.get_nowait())
|
||||
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in batch:
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished(blob)
|
||||
self.cleanup_active()
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
blob.close()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
except (OSError, Exception) as e:
|
||||
log.exception(e)
|
||||
raise e
|
||||
|
||||
def close(self):
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
downloader.close()
|
||||
finally:
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
@@ -146,26 +146,32 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
return None, response_msg
|
||||
break
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
|
||||
@@ -57,13 +57,8 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
@@ -77,7 +72,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
request = BlobRequest.deserialize(data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
@@ -100,7 +95,6 @@ class BlobServer:
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -108,7 +102,7 @@ class BlobServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
|
||||
+64
-63
@@ -16,11 +16,29 @@ log = logging.getLogger(__name__)
|
||||
NOT_SET = type(str('NOT_SET'), (object,), {})
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
API_ADDRESS = 'lbryapi'
|
||||
APP_NAME = 'LBRY'
|
||||
BLOBFILES_DIR = 'blobfiles'
|
||||
CRYPTSD_FILE_EXTENSION = '.cryptsd'
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
|
||||
LOG_FILE_NAME = 'lbrynet.log'
|
||||
LOG_POST_URL = 'https://lbry.io/log-upload'
|
||||
MAX_BLOB_REQUEST_SIZE = 64 * KB
|
||||
MAX_HANDSHAKE_SIZE = 64 * KB
|
||||
MAX_REQUEST_SIZE = 64 * KB
|
||||
MAX_RESPONSE_INFO_SIZE = 64 * KB
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
@@ -453,7 +471,7 @@ class CLIConfig(BaseConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
# directories
|
||||
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
@@ -464,77 +482,60 @@ class Config(CLIConfig):
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
# claims set to expire within this many blocks will be
|
||||
# automatically renewed after startup (if set to 0, renews
|
||||
# will not be made automatically)
|
||||
auto_renew_claim_height_delta = Integer("", 0)
|
||||
cache_time = Integer("", 150)
|
||||
data_rate = Float("points/megabyte", .0001)
|
||||
delete_blobs_on_remove = Toggle("", True)
|
||||
dht_node_port = Integer("", 4444)
|
||||
download_timeout = Float("", 30.0)
|
||||
blob_download_timeout = Float("", 20.0)
|
||||
peer_connect_timeout = Float("", 3.0)
|
||||
node_rpc_timeout = Float("", constants.rpc_timeout)
|
||||
is_generous_host = Toggle("", True)
|
||||
announce_head_blobs_only = Toggle("", True)
|
||||
concurrent_announcers = Integer("", 10)
|
||||
known_dht_nodes = Servers("", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
max_connections_per_stream = Integer("", 5)
|
||||
seek_head_blob_first = Toggle("", True)
|
||||
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
|
||||
min_info_rate = Float("points/1000 infos", .02)
|
||||
min_valuable_hash_rate = Float("points/1000 infos", .05)
|
||||
min_valuable_info_rate = Float("points/1000 infos", .05)
|
||||
peer_port = Integer("", 3333)
|
||||
pointtrader_server = String("", 'http://127.0.0.1:2424')
|
||||
reflector_port = Integer("", 5566)
|
||||
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
reflect_uploads = Toggle("", True)
|
||||
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
|
||||
reflector_servers = Servers("", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
|
||||
sd_download_timeout = Integer("", 3)
|
||||
peer_search_timeout = Integer("", 60)
|
||||
use_upnp = Toggle("", True)
|
||||
use_keyring = Toggle("", False)
|
||||
blockchain_name = String("", 'lbrycrd_main')
|
||||
lbryum_servers = Servers("", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -19,8 +19,6 @@ class BlobAnnouncer:
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
@@ -41,7 +39,7 @@ class BlobAnnouncer:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
await self.storage.update_last_announced_blobs(announced, self.loop.time())
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
|
||||
+33
-62
@@ -1,9 +1,9 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import socket
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -20,10 +20,10 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -64,7 +64,7 @@ class Node:
|
||||
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
@@ -79,9 +79,10 @@ class Node:
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
log.info("Store to %i peers", len(peers))
|
||||
log.info(peers)
|
||||
for peer in peers:
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
@@ -115,50 +116,39 @@ class Node:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
known_node_addresses = known_node_addresses or []
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses:
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
info = await self.loop.getaddrinfo(
|
||||
host, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
if (info[0][4][0], port) not in known_node_addresses:
|
||||
known_node_addresses.append((info[0][4][0], port))
|
||||
futs = []
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
futs.append(peer.ping())
|
||||
if futs:
|
||||
await asyncio.wait(futs, loop=self.loop)
|
||||
|
||||
if known_node_addresses:
|
||||
while not self.protocol.routing_table.get_peers():
|
||||
success = False
|
||||
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await peer.ping()
|
||||
success = True
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
|
||||
round(self.protocol.rpc_timeout, 2))
|
||||
if success:
|
||||
break
|
||||
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
|
||||
# in the routing table with good peers who are near us
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
self.joined.set()
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
@@ -224,32 +214,13 @@ class Node:
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(node_id, max_results=max_results,
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
log.info("peer search: %s", peers)
|
||||
accumulated.extend(peers)
|
||||
log.info("junction done")
|
||||
log.info("context done")
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
try:
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait([
|
||||
peer for peer in peers
|
||||
if not (
|
||||
peer.address == self.protocol.external_ip
|
||||
and peer.tcp_port == self.protocol.peer_port
|
||||
)
|
||||
])
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue()
|
||||
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
+54
-33
@@ -30,30 +30,37 @@ class PeerManager:
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
self._lock = asyncio.Lock(loop=loop)
|
||||
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
async def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
async with self._lock:
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
async def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
async with self._lock:
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
async def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
async with self._lock:
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
async def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
async with self._lock:
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
async def clear_token(self, node_id: bytes):
|
||||
async with self._lock:
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
async def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
async with self._lock:
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
@@ -63,36 +70,50 @@ class PeerManager:
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
|
||||
return self._node_id_mapping.get((address, udp_port))
|
||||
|
||||
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
return self._node_id_reverse_mapping.get(node_id)
|
||||
|
||||
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
async with self._lock:
|
||||
addr_tuple = self._node_id_reverse_mapping.get(node_id)
|
||||
if addr_tuple and addr_tuple in self._node_id_mapping:
|
||||
return addr_tuple
|
||||
|
||||
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
async with self._lock:
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
async def prune(self):
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
async with self._lock:
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
|
||||
@@ -100,6 +100,6 @@ class AsyncGeneratorJunction:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -1,9 +1,7 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
@@ -92,8 +90,8 @@ class IterativeFinder:
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
|
||||
self.active: typing.List['KademliaPeer'] = []
|
||||
self.contacted: typing.List[typing.Tuple[str, int]] = []
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
@@ -101,12 +99,14 @@ class IterativeFinder:
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.running_probes: typing.List[asyncio.Task] = []
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
@@ -114,16 +114,9 @@ class IterativeFinder:
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
Called with a lock after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
@@ -136,7 +129,9 @@ class IterativeFinder:
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
if not self.closest_peer:
|
||||
return True
|
||||
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
@@ -146,18 +141,21 @@ class IterativeFinder:
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.add(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
async with self.lock:
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.append(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
addr_tuple = (contact_triple[1], contact_triple[2])
|
||||
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(
|
||||
contact_triple[0], contact_triple[1], contact_triple[2]
|
||||
)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
@@ -165,11 +163,13 @@ class IterativeFinder:
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
return
|
||||
except RemoteException:
|
||||
return
|
||||
@@ -181,35 +181,31 @@ class IterativeFinder:
|
||||
"""
|
||||
|
||||
added = 0
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.add((peer.address, peer.udp_port))
|
||||
async with self.lock:
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.append((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
def callback(_):
|
||||
if t and t in self.running_probes:
|
||||
self.running_probes.remove(t)
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.append(t)
|
||||
added += 1
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
@@ -219,40 +215,70 @@ class IterativeFinder:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
drain_tasks(self.running_probes)
|
||||
self.running = False
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def __aiter__(self):
|
||||
def search(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
|
||||
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
|
||||
peers = self.loop.create_task(self.iteration_queue.get())
|
||||
finished = self.loop.create_task(self.finished.wait())
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
if peers.done():
|
||||
return peers.result()
|
||||
raise StopAsyncIteration()
|
||||
except asyncio.CancelledError as error:
|
||||
err = error
|
||||
finally:
|
||||
if not finished.done() and not finished.cancelled():
|
||||
finished.cancel()
|
||||
if not peers.done() and not peers.cancelled():
|
||||
peers.cancel()
|
||||
if err:
|
||||
raise err
|
||||
|
||||
def __aiter__(self):
|
||||
self.search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
initial_results = self.get_initial_result()
|
||||
if initial_results:
|
||||
self.iteration_queue.put_nowait(initial_results)
|
||||
result = await self.next_queue_or_finished()
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
self.loop.call_soon(self.aclose)
|
||||
await self.aclose()
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
async def _aclose():
|
||||
async with self.lock:
|
||||
self.running = False
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
drain_tasks(self.tasks)
|
||||
drain_tasks(self.running_probes)
|
||||
while self.delayed_calls:
|
||||
timer = self.delayed_calls.pop()
|
||||
if timer:
|
||||
timer.cancel()
|
||||
|
||||
return asyncio.ensure_future(_aclose(), loop=self.loop)
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
@@ -269,26 +295,24 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
|
||||
def put_result(self, from_list: typing.List['KademliaPeer']):
|
||||
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.shortlist, finish=True)
|
||||
log.info("found")
|
||||
self.put_result(self.shortlist)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
@@ -299,10 +323,16 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.debug("max results")
|
||||
self.put_result(self.active, True)
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.info("max results")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
@@ -336,11 +366,14 @@ class IterativeValueFinder(IterativeFinder):
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
self.iteration_queue.put_nowait(None)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
@@ -77,23 +77,24 @@ class KademliaRPC:
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'contacts': self.find_node(rpc_contact, key)
|
||||
}
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
# get peers we have stored for this blob_exchange
|
||||
peers = [
|
||||
peer.compact_address_tcp()
|
||||
for peer in self.protocol.data_store.get_peers_for_blob(key)
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
@@ -179,7 +180,7 @@ class RemoteKademliaRPC:
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
@@ -192,21 +193,23 @@ class PingQueue:
|
||||
self._process_task: asyncio.Task = None
|
||||
self._next_task: asyncio.Future = None
|
||||
self._next_timer: asyncio.TimerHandle = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._running = False
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
async def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = constants.check_refresh_interval if delay is None else delay
|
||||
for peer in peers:
|
||||
if delay and peer not in self._enqueued_contacts:
|
||||
self._pending_contacts[peer] = self._loop.time() + delay
|
||||
elif peer not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(peer)
|
||||
if peer in self._pending_contacts:
|
||||
del self._pending_contacts[peer]
|
||||
async with self._lock:
|
||||
for peer in peers:
|
||||
if delay and peer not in self._enqueued_contacts:
|
||||
self._pending_contacts[peer] = self._loop.time() + delay
|
||||
elif peer not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(peer)
|
||||
if peer in self._pending_contacts:
|
||||
del self._pending_contacts[peer]
|
||||
|
||||
async def _process(self):
|
||||
async def _ping(p: 'KademliaPeer'):
|
||||
@@ -215,22 +218,23 @@ class PingQueue:
|
||||
await self._protocol.add_peer(p)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
except asyncio.TimeoutError:
|
||||
except TimeoutError:
|
||||
pass
|
||||
|
||||
while True:
|
||||
tasks = []
|
||||
|
||||
if self._enqueued_contacts or self._pending_contacts:
|
||||
now = self._loop.time()
|
||||
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
|
||||
for k in scheduled:
|
||||
del self._pending_contacts[k]
|
||||
if k not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(k)
|
||||
while self._enqueued_contacts:
|
||||
peer = self._enqueued_contacts.pop()
|
||||
tasks.append(self._loop.create_task(_ping(peer)))
|
||||
async with self._lock:
|
||||
if self._enqueued_contacts or self._pending_contacts:
|
||||
now = self._loop.time()
|
||||
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
|
||||
for k in scheduled:
|
||||
del self._pending_contacts[k]
|
||||
if k not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(k)
|
||||
while self._enqueued_contacts:
|
||||
peer = self._enqueued_contacts.pop()
|
||||
tasks.append(self._loop.create_task(_ping(peer)))
|
||||
if tasks:
|
||||
await asyncio.wait(tasks, loop=self._loop)
|
||||
|
||||
@@ -279,6 +283,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
@@ -410,8 +415,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
await self.peer_manager.report_last_requested(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
@@ -420,18 +425,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
await self.ping_queue.enqueue_maybe_ping(peer)
|
||||
elif is_good is True:
|
||||
await self.add_peer(peer)
|
||||
except ValueError as err:
|
||||
log.debug("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
await self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
|
||||
except Exception as err:
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
@@ -460,8 +457,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
await self.peer_manager.report_last_replied(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
@@ -478,23 +475,21 @@ class KademliaProtocol(DatagramProtocol):
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
)
|
||||
return
|
||||
|
||||
error_msg = f"" \
|
||||
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
|
||||
f"Args: {request.args}\n" \
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if 'Invalid token' in error_msg:
|
||||
log.debug(error_msg)
|
||||
elif error_datagram.response not in old_protocol_errors:
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
|
||||
# reject replies coming from a different address than what we sent our request to
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
log.error("node id mismatch in reply")
|
||||
remote_exception = TimeoutError(peer.node_id)
|
||||
df.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
@@ -510,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -527,10 +522,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.routing_table.remove_peer(peer)
|
||||
raise
|
||||
@@ -559,29 +554,30 @@ class KademliaProtocol(DatagramProtocol):
|
||||
if message.rpc_id in self.sent_messages:
|
||||
self.sent_messages.pop(message.rpc_id)
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
async with self.lock:
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
@@ -605,7 +601,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
@@ -613,7 +609,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
await self.peer_manager.clear_token(peer.node_id)
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
@@ -183,6 +183,8 @@ class TreeRoutingTable:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
if key in exclude:
|
||||
exclude.remove(key)
|
||||
count = count or constants.k
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
|
||||
+2
-16
@@ -1,6 +1,4 @@
|
||||
import os
|
||||
import sys
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -218,19 +216,12 @@ def get_argument_parser():
|
||||
return main
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
@@ -247,23 +238,17 @@ def main(argv=None):
|
||||
)
|
||||
file_handler.setFormatter(default_formatter)
|
||||
log.addHandler(file_handler)
|
||||
logging.getLogger('torba').addHandler(file_handler)
|
||||
|
||||
if not args.quiet:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
log.addHandler(handler)
|
||||
logging.getLogger('torba').addHandler(handler)
|
||||
logging.getLogger('torba').setLevel(logging.INFO)
|
||||
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
loop.set_debug(True)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
if conf.share_usage_data:
|
||||
@@ -272,9 +257,10 @@ def main(argv=None):
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
daemon = Daemon(conf)
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(daemon.start())
|
||||
loop.run_until_complete(daemon.stop_event.wait())
|
||||
loop.run_forever()
|
||||
except (GracefulExit, KeyboardInterrupt):
|
||||
pass
|
||||
finally:
|
||||
|
||||
@@ -5,6 +5,7 @@ import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import socket
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
import base58
|
||||
@@ -17,6 +18,7 @@ import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
@@ -63,6 +65,14 @@ async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
pass
|
||||
|
||||
|
||||
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
|
||||
info = await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
return info[0][4][0]
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -106,7 +116,7 @@ class DatabaseComponent(Component):
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
@@ -321,7 +331,9 @@ class BlobComponent(Component):
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
self.blob_manager.stop()
|
||||
while self.blob_manager and self.blob_manager.blobs:
|
||||
_, blob = self.blob_manager.blobs.popitem()
|
||||
await blob.close()
|
||||
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
@@ -347,7 +359,7 @@ class DHTComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
@@ -364,8 +376,8 @@ class DHTComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
@@ -377,14 +389,13 @@ class DHTComponent(Component):
|
||||
asyncio.get_event_loop(),
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
internal_udp_port=self.conf.dht_node_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout
|
||||
peer_port=self.external_peer_port
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
|
||||
)
|
||||
log.info("Started the dht")
|
||||
|
||||
@@ -408,7 +419,7 @@ class HashAnnouncerComponent(Component):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
self.hash_announcer.start(self.conf.concurrent_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
async def stop(self):
|
||||
@@ -451,13 +462,17 @@ class StreamManagerComponent(Component):
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
|
||||
self.conf.peer_connect_timeout, [
|
||||
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
|
||||
for url, port in self.conf.reflector_servers
|
||||
], self.conf.reflector_servers
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
await self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -477,10 +492,10 @@ class PeerProtocolServerComponent(Component):
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
self.blob_server.start_server(peer_port, interface='0.0.0.0')
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
@@ -493,8 +508,8 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
@@ -599,9 +614,7 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if self.component_manager.analytics_manager:
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
@@ -627,7 +640,7 @@ class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
Component.__init__(self, component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
|
||||
+127
-132
@@ -6,10 +6,10 @@ import inspect
|
||||
import typing
|
||||
import aiohttp
|
||||
import base58
|
||||
import random
|
||||
from urllib.parse import urlencode, quote
|
||||
from typing import Callable, Optional, List
|
||||
from binascii import hexlify, unhexlify
|
||||
from copy import deepcopy
|
||||
from traceback import format_exc
|
||||
from aiohttp import web
|
||||
from functools import wraps
|
||||
@@ -18,9 +18,9 @@ from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
|
||||
from lbrynet import __version__, utils
|
||||
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash
|
||||
from lbrynet.blob_exchange.downloader import download_blob
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
|
||||
from lbrynet.extras import system_info
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
@@ -29,7 +29,6 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
|
||||
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.extras.daemon.undecorated import undecorated
|
||||
from lbrynet.extras.wallet.account import Account as LBCAccount
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -256,8 +255,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
skip_components=conf.components_to_skip or []
|
||||
)
|
||||
self.component_startup_task = None
|
||||
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
|
||||
self.stop_event = asyncio.Event()
|
||||
|
||||
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
|
||||
app = web.Application()
|
||||
@@ -370,18 +367,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not os.path.isdir(self.conf.download_dir):
|
||||
os.makedirs(self.conf.download_dir)
|
||||
|
||||
async def update_connection_status(self):
|
||||
connected = await utils.async_check_connection()
|
||||
self._connection_status = (self.component_manager.loop.time(), connected)
|
||||
|
||||
async def get_connection_status(self) -> str:
|
||||
if self._connection_status[0] + 300 > self.component_manager.loop.time():
|
||||
if not self._connection_status[1]:
|
||||
await self.update_connection_status()
|
||||
else:
|
||||
await self.update_connection_status()
|
||||
return CONNECTION_STATUS_CONNECTED if self._connection_status[1] else CONNECTION_STATUS_NETWORK
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting LBRYNet Daemon")
|
||||
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
|
||||
@@ -605,7 +590,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(string) Shutdown message
|
||||
"""
|
||||
log.info("Shutting down lbrynet daemon")
|
||||
self.stop_event.set()
|
||||
return "Shutting down"
|
||||
|
||||
async def jsonrpc_status(self):
|
||||
@@ -680,8 +664,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
connection_code = await self.get_connection_status()
|
||||
|
||||
connection_code = CONNECTION_STATUS_NETWORK
|
||||
response = {
|
||||
'installation_id': self.installation_id,
|
||||
'is_running': all(self.component_manager.get_components_status().values()),
|
||||
@@ -1324,9 +1307,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
|
||||
[--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
|
||||
[--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
|
||||
[--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>]
|
||||
[--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>]
|
||||
[--comparison=<comparison>] [--full_status=<full_status>] [--reverse]
|
||||
[--claim_name=<claim_name>] [--sort=<sort_by>] [--reverse] [--comparison=<comparison>]
|
||||
[--full_status=<full_status>]
|
||||
|
||||
Options:
|
||||
--sd_hash=<sd_hash> : (str) get file with matching sd hash
|
||||
@@ -1339,12 +1321,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str) get file with matching claim txid
|
||||
--nout=<nout> : (int) get file with matching claim nout
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--claim_name=<claim_name> : (str) get file with matching claim name
|
||||
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
|
||||
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
|
||||
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
|
||||
--comparison=<comparison> : (str) logical comparision, (eq | ne | g | ge | l | le)
|
||||
--sort=<sort_method> : (str) sort by any property, like 'file_name'
|
||||
or 'metadata.author'; to specify direction
|
||||
append ',asc' or ',desc'
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
@@ -1363,26 +1344,23 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes_lower_bound': (int) lower bound file size in bytes,
|
||||
'total_bytes': (int) file upper bound size in bytes,
|
||||
'total_bytes': (int) file size in bytes,
|
||||
'written_bytes': (int) written size in bytes,
|
||||
'blobs_completed': (int) number of fully downloaded blobs,
|
||||
'blobs_in_stream': (int) total blobs on stream,
|
||||
'blobs_remaining': (int) total blobs remaining to download,
|
||||
'status': (str) downloader status
|
||||
'claim_id': (str) None if claim is not found else the claim id,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'txid': (str) None if claim is not found else the transaction id,
|
||||
'nout': (int) None if claim is not found else the transaction output index,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'metadata': (dict) None if claim is not found else the claim metadata,
|
||||
'channel_claim_id': (str) None if claim is not found or not signed,
|
||||
'channel_name': (str) None if claim is not found or not signed,
|
||||
'claim_name': (str) None if claim is not found else the claim name
|
||||
},
|
||||
]
|
||||
}
|
||||
"""
|
||||
sort = sort or 'rowid'
|
||||
sort = sort or 'status'
|
||||
comparison = comparison or 'eq'
|
||||
return [
|
||||
stream.as_dict() for stream in self.stream_manager.get_filtered_streams(
|
||||
@@ -1439,23 +1417,25 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return claim_results
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_resolve(self, urls: typing.Union[str, list]):
|
||||
async def jsonrpc_resolve(self, force=False, uri=None, uris=None):
|
||||
"""
|
||||
Get the claim that a URL refers to.
|
||||
Resolve given LBRY URIs
|
||||
|
||||
Usage:
|
||||
resolve <urls>...
|
||||
resolve [--force] (<uri> | --uri=<uri>) [<uris>...]
|
||||
|
||||
Options:
|
||||
--urls=<urls> : (str, list) one or more urls to resolve
|
||||
--force : (bool) force refresh and ignore cache
|
||||
--uri=<uri> : (str) uri to resolve
|
||||
--uris=<uris> : (list) uris to resolve
|
||||
|
||||
Returns:
|
||||
Dictionary of results, keyed by url
|
||||
'<url>': {
|
||||
Dictionary of results, keyed by uri
|
||||
'<uri>': {
|
||||
If a resolution error occurs:
|
||||
'error': Error message
|
||||
|
||||
If the url resolves to a channel or a claim in a channel:
|
||||
If the uri resolves to a channel or a claim in a channel:
|
||||
'certificate': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
@@ -1477,10 +1457,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
|
||||
If the url resolves to a channel:
|
||||
If the uri resolves to a channel:
|
||||
'claims_in_channel': (int) number of claims in the channel,
|
||||
|
||||
If the url resolves to a claim:
|
||||
If the uri resolves to a claim:
|
||||
'claim': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
@@ -1505,24 +1485,23 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
if isinstance(urls, str):
|
||||
urls = [urls]
|
||||
uris = tuple(uris or [])
|
||||
if uri is not None:
|
||||
uris += (uri,)
|
||||
|
||||
results = {}
|
||||
|
||||
valid_urls = set()
|
||||
for u in urls:
|
||||
valid_uris = tuple()
|
||||
for u in uris:
|
||||
try:
|
||||
parse_lbry_uri(u)
|
||||
valid_urls.add(u)
|
||||
valid_uris += (u,)
|
||||
except URIParseError:
|
||||
results[u] = {"error": "%s is not a valid url" % u}
|
||||
|
||||
resolved = await self.wallet_manager.resolve(*tuple(valid_urls))
|
||||
results[u] = {"error": "%s is not a valid uri" % u}
|
||||
|
||||
resolved = await self.wallet_manager.resolve(*valid_uris, check_cache=not force)
|
||||
for resolved_uri in resolved:
|
||||
results[resolved_uri] = resolved[resolved_uri]
|
||||
|
||||
return results
|
||||
|
||||
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT,
|
||||
@@ -1572,17 +1551,44 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
try:
|
||||
stream = await self.stream_manager.download_stream_from_uri(
|
||||
uri, self.exchange_rate_manager, file_name, timeout
|
||||
timeout = timeout if timeout is not None else self.conf.download_timeout
|
||||
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise Exception("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet_manager.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
)
|
||||
if not stream:
|
||||
raise DownloadSDTimeout(uri)
|
||||
except Exception as e:
|
||||
log.warning("Error downloading %s: %s", uri, str(e))
|
||||
return {"error": str(e)}
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(self.exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
fee_address = claim.source_fee.address
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.stream_manager.get_filtered_streams(outpoint=outpoint)
|
||||
if not existing:
|
||||
existing.extend(self.stream_manager.get_filtered_streams(claim_id=resolved['claim_id'],
|
||||
sd_hash=claim.source_hash))
|
||||
if existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
else:
|
||||
stream = await self.stream_manager.download_stream_from_claim(
|
||||
self.dht_node, self.conf.download_dir, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
if stream:
|
||||
return stream.as_dict()
|
||||
raise DownloadSDTimeout(resolved['value']['stream']['source']['source'])
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_set_status(self, status, **kwargs):
|
||||
@@ -1612,11 +1618,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not streams:
|
||||
raise Exception(f'Unable to find a file for {kwargs}')
|
||||
stream = streams[0]
|
||||
if status == 'start' and not stream.running:
|
||||
await self.stream_manager.start_stream(stream)
|
||||
if status == 'start' and not stream.running and not stream.finished:
|
||||
stream.downloader.download(self.dht_node)
|
||||
msg = "Resumed download"
|
||||
elif status == 'stop' and stream.running:
|
||||
await self.stream_manager.stop_stream(stream)
|
||||
await stream.stop_download()
|
||||
msg = "Stopped download"
|
||||
else:
|
||||
msg = (
|
||||
@@ -1823,7 +1829,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def jsonrpc_publish(
|
||||
self, name, bid, metadata=None, file_path=None, fee=None, title=None,
|
||||
description=None, author=None, language=None, license=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
|
||||
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
|
||||
claim_address=None, change_address=None):
|
||||
"""
|
||||
@@ -1847,7 +1853,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
|
||||
[--description=<description>] [--author=<author>] [--language=<language>]
|
||||
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
|
||||
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
|
||||
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
|
||||
[--claim_address=<claim_address>] [--change_address=<change_address>]
|
||||
@@ -1883,6 +1889,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--thumbnail=<thumbnail> : (str) thumbnail url
|
||||
--preview=<preview> : (str) preview url
|
||||
--nsfw=<nsfw> : (bool) whether the content is nsfw
|
||||
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
|
||||
for channel claim being in the wallet. This allows
|
||||
@@ -1906,9 +1913,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
try:
|
||||
parsed = parse_lbry_uri(name)
|
||||
if parsed.name != name:
|
||||
raise Exception("Name given to publish has invalid characters")
|
||||
parse_lbry_uri(name)
|
||||
except (TypeError, URIParseError):
|
||||
raise Exception("Invalid name given to publish")
|
||||
|
||||
@@ -1924,10 +1929,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
account = self.get_account_or_default(account_id)
|
||||
|
||||
available = await account.get_balance()
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) > 1:
|
||||
log.warning("found %i claims for the name %s", len(existing_claims), name)
|
||||
existing_claims = []
|
||||
if amount >= available:
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) == 1:
|
||||
available += existing_claims[0].get_estimator(self.ledger).effective_amount
|
||||
if amount >= available:
|
||||
@@ -1986,62 +1990,39 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
}
|
||||
|
||||
sd_to_delete = None
|
||||
|
||||
if file_path:
|
||||
# this will be used to verify the format with lbrynet.schema
|
||||
claim_copy = deepcopy(claim_dict)
|
||||
if sources is not None:
|
||||
claim_dict['stream']['source'] = sources
|
||||
claim_copy['stream']['source'] = sources
|
||||
elif file_path is not None:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("invalid file path to publish")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
if existing_claims:
|
||||
sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source']
|
||||
|
||||
# since the file hasn't yet been made into a stream, we don't have
|
||||
# a valid Source for the claim when validating the format, we'll use a fake one
|
||||
claim_dict['stream']['source'] = {
|
||||
claim_copy['stream']['source'] = {
|
||||
'version': '_0_0_1',
|
||||
'sourceType': 'lbry_sd_hash',
|
||||
'source': '0' * 96,
|
||||
'contentType': ''
|
||||
}
|
||||
elif not existing_claims:
|
||||
raise Exception("no previous stream to update")
|
||||
else:
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
# there is no existing source to use, and a file was not provided to make a new one
|
||||
raise Exception("no source provided to publish")
|
||||
try:
|
||||
claim_dict = ClaimDict.load_dict(claim_dict).claim_dict
|
||||
ClaimDict.load_dict(claim_copy)
|
||||
# the metadata to use in the claim can be serialized by lbrynet.schema
|
||||
except DecodeError as err:
|
||||
# there was a problem with a metadata field, raise an error here rather than
|
||||
# waiting to find out when we go to publish the claim (after having made the stream)
|
||||
raise Exception(f"invalid publish metadata: {err}")
|
||||
|
||||
certificate = None
|
||||
if channel_id or channel_name:
|
||||
certificate = await self.get_channel_or_error(
|
||||
self.get_accounts_or_all(channel_account_id), channel_id, channel_name
|
||||
)
|
||||
|
||||
if file_path:
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
|
||||
if sd_to_delete:
|
||||
stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete)
|
||||
stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete)
|
||||
if stream_to_delete:
|
||||
await self.stream_manager.delete_stream(stream_to_delete, delete_file=False)
|
||||
log.info("updating claim to stream generated from %s, deleted previous stream %s",
|
||||
sd_to_delete[:8], file_path)
|
||||
else:
|
||||
log.info("previous stream %s from claim was not saved locally, nothing to delete",
|
||||
sd_to_delete[:8])
|
||||
else:
|
||||
log.info("generated stream from %s for claim", file_path)
|
||||
else:
|
||||
log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8])
|
||||
|
||||
sd_hash = claim_dict['stream']['source']['source']
|
||||
log.info("Publish: %s", {
|
||||
'name': name,
|
||||
'file_path': file_path,
|
||||
@@ -2052,14 +2033,35 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'channel_id': channel_id,
|
||||
'channel_name': channel_name
|
||||
})
|
||||
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File {file_path} not found")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
claim_dict['stream']['source'] = {}
|
||||
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
stream_hash = stream.stream_hash
|
||||
await self.storage.save_published_file(stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
else:
|
||||
if not ('source' not in claim_dict['stream'] and existing_claims):
|
||||
raise Exception("no previous stream to update")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source'])
|
||||
tx = await self.wallet_manager.claim_name(
|
||||
account, name, amount, claim_dict, certificate, claim_address
|
||||
)
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
if stream_hash:
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
nout = 0
|
||||
txo = tx.outputs[nout]
|
||||
@@ -2287,7 +2289,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
claims = await self.wallet_manager.get_claims_for_name(name) # type: dict
|
||||
claims['claims'] = sort_claim_results(claims['claims'])
|
||||
sort_claim_results(claims['claims'])
|
||||
return claims
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -2567,7 +2569,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
"""
|
||||
|
||||
blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash)
|
||||
downloader = BlobDownloader(asyncio.get_event_loop(), self.blob_manager, self.conf)
|
||||
blob = await downloader.get_blob(blob_hash, self.dht_node)
|
||||
if read:
|
||||
with open(blob.file_path, 'rb') as handle:
|
||||
return handle.read().decode()
|
||||
@@ -2788,19 +2791,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(list) list of blobs reflected
|
||||
"""
|
||||
|
||||
server, port = kwargs.get('server'), kwargs.get('port')
|
||||
if server and port:
|
||||
port = int(port)
|
||||
else:
|
||||
server, port = random.choice(self.conf.reflector_servers)
|
||||
reflected = await asyncio.gather(*[
|
||||
stream.upload_to_reflector(server, port)
|
||||
for stream in self.stream_manager.get_filtered_streams(**kwargs)
|
||||
])
|
||||
total = []
|
||||
for reflected_for_stream in reflected:
|
||||
total.extend(reflected_for_stream)
|
||||
return total
|
||||
raise NotImplementedError()
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
async def jsonrpc_peer_ping(self, node_id, address, port):
|
||||
@@ -2816,14 +2807,18 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
peer = None
|
||||
if node_id and address and port:
|
||||
peer = self.component_manager.peer_manager.get_kademlia_peer(unhexlify(node_id), address,
|
||||
udp_port=int(port))
|
||||
try:
|
||||
return await self.dht_node.protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
return {'error': 'timeout'}
|
||||
peer = self.component_manager.peer_manager.get_peer(address, unhexlify(node_id), udp_port=int(port))
|
||||
if not peer:
|
||||
peer = self.component_manager.peer_manager.make_peer(
|
||||
address, unhexlify(node_id), udp_port=int(port)
|
||||
)
|
||||
if not peer:
|
||||
return {'error': 'peer not found'}
|
||||
try:
|
||||
result = await peer.ping()
|
||||
return result.decode()
|
||||
except TimeoutError:
|
||||
return {'error': 'ping timeout'}
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
def jsonrpc_routing_table_get(self):
|
||||
|
||||
@@ -5,12 +5,9 @@ import logging
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
|
||||
@@ -219,7 +219,7 @@ class ExchangeRateManager:
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
if from_currency == to_currency:
|
||||
return amount
|
||||
|
||||
|
||||
@@ -154,7 +154,5 @@ types_map = {
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(
|
||||
ext,
|
||||
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
|
||||
)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
+107
-193
@@ -1,10 +1,8 @@
|
||||
import os
|
||||
import logging
|
||||
import sqlite3
|
||||
import typing
|
||||
import asyncio
|
||||
import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -106,104 +104,14 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
|
||||
return StoredStreamClaim(*claim_fields)
|
||||
|
||||
|
||||
def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
for start_index in range(0, len(parameters), batch_size):
|
||||
current_batch = parameters[start_index:start_index+batch_size]
|
||||
def _batched_select(transaction, query, parameters):
|
||||
for start_index in range(0, len(parameters), 900):
|
||||
current_batch = parameters[start_index:start_index+900]
|
||||
bind = "({})".format(','.join(['?'] * len(current_batch)))
|
||||
for result in transaction.execute(query.format(bind), current_batch):
|
||||
yield result
|
||||
|
||||
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", [
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
|
||||
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
signed_claims[claim.channel_claim_id] = []
|
||||
signed_claims[claim.channel_claim_id].append(claim)
|
||||
files.append(
|
||||
{
|
||||
"rowid": rowid,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name, # hex
|
||||
"download_directory": download_dir, # hex
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim
|
||||
}
|
||||
)
|
||||
for claim_id in signed_claims.keys():
|
||||
channel_name = transaction.execute("select claim_name from claim where claim_id=?", (claim_id, )).fetchone()
|
||||
if channel_name:
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = channel_name[0]
|
||||
return files
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
|
||||
data_payment_rate: float, status: str) -> int:
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
)
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
@@ -276,12 +184,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
|
||||
def __init__(self, conf: Config, path, loop=None):
|
||||
super().__init__(path)
|
||||
self.conf = conf
|
||||
self.content_claim_callbacks = {}
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
self.time_getter = time_getter or time.time
|
||||
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
@@ -295,16 +202,15 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash: str, length: int):
|
||||
def _add_blob(transaction: sqlite3.Connection):
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
transaction.execute(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
|
||||
)
|
||||
return self.db.run(_add_blob)
|
||||
def add_completed_blob(self, blob_hash: str):
|
||||
log.debug("Adding a completed blob. blob_hash=%s", blob_hash)
|
||||
return self.db.execute("update blob set status='finished' where blob.blob_hash=?", (blob_hash, ))
|
||||
|
||||
def set_should_announce(self, blob_hash: str, next_announce_time: int, should_announce: int):
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
|
||||
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
|
||||
)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -341,9 +247,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select count(*) from blob where status='finished'"
|
||||
)
|
||||
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str], last_announced: float):
|
||||
def _update_last_announced_blobs(transaction: sqlite3.Connection):
|
||||
last_announced = self.time_getter()
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
@@ -354,7 +259,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = int(self.time_getter())
|
||||
now = self.loop.time()
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
@@ -369,23 +274,21 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = int(self.time_getter())
|
||||
if self.conf.announce_head_and_sd_only:
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished'",
|
||||
(timestamp,)
|
||||
)
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
"and next_announce_time<? and status='finished'", (timestamp,)
|
||||
)
|
||||
return [b[0] for b in r.fetchall()]
|
||||
blobs = [b[0] for b in r.fetchall()]
|
||||
return blobs
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
@@ -393,31 +296,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
|
||||
)
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
return self.db.run(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
to_update = [
|
||||
(blob_hash, )
|
||||
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
|
||||
if blob_hash not in blob_files
|
||||
]
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
to_update
|
||||
)
|
||||
return {
|
||||
blob_hash
|
||||
for blob_hash, in _batched_select(
|
||||
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
|
||||
list(blob_files)
|
||||
)
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
@@ -430,20 +313,42 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
|
||||
return streams is not None
|
||||
|
||||
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
|
||||
return self.run_and_return_one_or_none(
|
||||
"select rowid from file where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
return self.db.run(store_stream, sd_blob, descriptor)
|
||||
def _store_stream(transaction: sqlite3.Connection):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]:
|
||||
return self.db.run(_store_stream)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False):
|
||||
def _get_blobs_for_stream(transaction):
|
||||
crypt_blob_infos = []
|
||||
stream_blobs = transaction.execute(
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=? "
|
||||
"order by position asc", (stream_hash, )
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=?", (stream_hash, )
|
||||
).fetchall()
|
||||
if only_completed:
|
||||
lengths = transaction.execute(
|
||||
@@ -465,8 +370,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for blob_hash, position, iv in stream_blobs:
|
||||
blob_length = blob_length_dict.get(blob_hash, 0)
|
||||
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
|
||||
if not blob_hash:
|
||||
break
|
||||
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
|
||||
return crypt_blob_infos
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
@@ -481,58 +385,68 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
def _delete_stream(transaction: sqlite3.Connection):
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, ))
|
||||
transaction.executemany("delete from blob where blob_hash=?",
|
||||
[(blob.blob_hash, ) for blob in descriptor.blobs[:-1]])
|
||||
return self.db.run(_delete_stream)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory,
|
||||
data_payment_rate) -> typing.Awaitable[int]:
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
return self.save_published_file(
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
|
||||
status="running"
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished") -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
status="finished"):
|
||||
return self.db.execute(
|
||||
"insert into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
|
||||
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name) -> typing.Dict:
|
||||
return {
|
||||
"row_id": rowid,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name,
|
||||
"download_directory": download_dir,
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name,
|
||||
"suggested_file_name": suggested_file_name
|
||||
}
|
||||
|
||||
def _get_all_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
file_infos = list(map(lambda a: _lbry_file_dict(*a), transaction.execute(
|
||||
"select file.rowid, file.*, stream.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash"
|
||||
).fetchall()))
|
||||
stream_hashes = [file_info['stream_hash'] for file_info in file_infos]
|
||||
claim_infos = get_claims_from_stream_hashes(transaction, stream_hashes)
|
||||
for index in range(len(file_infos)): # pylint: disable=consider-using-enumerate
|
||||
file_infos[index]['claim'] = claim_infos.get(file_infos[index]['stream_hash'])
|
||||
return file_infos
|
||||
|
||||
results = await self.db.run(_get_all_files)
|
||||
if results:
|
||||
return results
|
||||
return []
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
log.info("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
|
||||
for descriptor, sd_blob in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped')
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
[(download_dir, stream_hash) for stream_hash in stream_hashes]
|
||||
)
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
@@ -768,7 +682,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
)
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
@@ -780,5 +694,5 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
int(self.time_getter()) - 86400
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
)
|
||||
@@ -260,6 +260,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
async def resolve(self, *uris, **kwargs):
|
||||
page = kwargs.get('page', 0)
|
||||
page_size = kwargs.get('page_size', 10)
|
||||
check_cache = kwargs.get('check_cache', False) # TODO: put caching back (was force_refresh parameter)
|
||||
ledger: MainNetLedger = self.default_account.ledger
|
||||
results = await ledger.resolve(page, page_size, *uris)
|
||||
if 'error' not in results:
|
||||
|
||||
+33
-68
@@ -15,13 +15,13 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
|
||||
base_name, ext = os.path.splitext(os.path.basename(file_name))
|
||||
base_name, ext = os.path.splitext(file_name)
|
||||
i = 0
|
||||
while os.path.isfile(os.path.join(download_directory, file_name)):
|
||||
i += 1
|
||||
file_name = "%s_%i%s" % (base_name, i, ext)
|
||||
|
||||
return file_name
|
||||
return os.path.join(download_directory, file_name)
|
||||
|
||||
|
||||
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
|
||||
@@ -29,9 +29,7 @@ async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_dir
|
||||
|
||||
|
||||
class StreamAssembler:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_file_name: typing.Optional[str] = None):
|
||||
self.output_file_name = output_file_name
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
@@ -45,11 +43,11 @@ class StreamAssembler:
|
||||
self.written_bytes: int = 0
|
||||
|
||||
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
|
||||
if not blob or not self.stream_handle or self.stream_handle.closed:
|
||||
return False
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
|
||||
def _decrypt_and_write():
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
if self.stream_handle.closed:
|
||||
return False
|
||||
self.stream_handle.seek(offset)
|
||||
_decrypted = blob.decrypt(
|
||||
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
|
||||
@@ -57,80 +55,47 @@ class StreamAssembler:
|
||||
self.stream_handle.write(_decrypted)
|
||||
self.stream_handle.flush()
|
||||
self.written_bytes += len(_decrypted)
|
||||
log.debug("decrypted %s", blob.blob_hash[:8])
|
||||
return True
|
||||
|
||||
await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
return True
|
||||
|
||||
async def setup(self):
|
||||
pass
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
pass
|
||||
|
||||
async def after_finished(self):
|
||||
pass
|
||||
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
if decrypted:
|
||||
log.info("decrypted %s", blob.blob_hash[:8])
|
||||
return
|
||||
|
||||
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
|
||||
if not os.path.isdir(output_dir):
|
||||
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
|
||||
await self.setup()
|
||||
self.sd_blob = await self.get_blob(self.sd_hash)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
|
||||
self.sd_blob)
|
||||
await self.after_got_descriptor()
|
||||
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
|
||||
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
|
||||
self.output_path = os.path.join(output_dir, self.output_file_name)
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
self.output_path = await get_next_available_file_name(self.loop, output_dir,
|
||||
output_file_name or self.descriptor.suggested_file_name)
|
||||
|
||||
self.stream_handle = open(self.output_path, 'wb')
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.sd_blob, self.descriptor
|
||||
)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
written_blobs = None
|
||||
save_tasks = []
|
||||
try:
|
||||
with open(self.output_path, 'wb') as stream_handle:
|
||||
self.stream_handle = stream_handle
|
||||
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
|
||||
if blob_info.blob_num != i:
|
||||
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
|
||||
return
|
||||
while self.stream_handle and not self.stream_handle.closed:
|
||||
try:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
if blob and blob.length != blob_info.length:
|
||||
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
|
||||
await self.blob_manager.delete_blobs([blob_info.blob_hash])
|
||||
continue
|
||||
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
|
||||
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
|
||||
written_blobs = i
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
|
||||
break
|
||||
except FileNotFoundError:
|
||||
log.debug("stream assembler stopped")
|
||||
return
|
||||
except (ValueError, IOError, OSError):
|
||||
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash)
|
||||
continue
|
||||
for blob_info in self.descriptor.blobs[:-1]:
|
||||
while True:
|
||||
try:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
|
||||
break
|
||||
except ValueError as err:
|
||||
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash, str(err))
|
||||
continue
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
self.stream_finished_event.set()
|
||||
finally:
|
||||
if written_blobs == len(self.descriptor.blobs) - 2:
|
||||
log.debug("finished decrypting and assembling stream")
|
||||
if save_tasks:
|
||||
await asyncio.wait(save_tasks)
|
||||
await self.after_finished()
|
||||
self.stream_finished_event.set()
|
||||
else:
|
||||
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
|
||||
len(self.descriptor.blobs) - 2)
|
||||
if self.output_path and os.path.isfile(self.output_path):
|
||||
log.debug("erasing incomplete file assembly: %s", self.output_path)
|
||||
os.unlink(self.output_path)
|
||||
self.stream_handle.close()
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
return self.blob_manager.get_blob(blob_hash, length)
|
||||
f = asyncio.Future(loop=self.loop)
|
||||
f.set_result(self.blob_manager.get_blob(blob_hash, length))
|
||||
return await f
|
||||
@@ -4,7 +4,6 @@ import binascii
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from lbrynet.blob import MAX_BLOB_SIZE
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
@@ -59,10 +58,6 @@ class StreamDescriptor:
|
||||
self.stream_hash = stream_hash or self.get_stream_hash()
|
||||
self.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return len(self.as_json())
|
||||
|
||||
def get_stream_hash(self) -> str:
|
||||
return self.calculate_stream_hash(
|
||||
binascii.hexlify(self.stream_name.encode()), self.key.encode(),
|
||||
@@ -83,46 +78,15 @@ class StreamDescriptor:
|
||||
[blob_info.as_dict() for blob_info in self.blobs]), sort_keys=True
|
||||
).encode()
|
||||
|
||||
def old_sort_json(self) -> bytes:
|
||||
blobs = []
|
||||
for b in self.blobs:
|
||||
blobs.append(OrderedDict(
|
||||
[('length', b.length), ('blob_num', b.blob_num), ('iv', b.iv)] if not b.blob_hash else
|
||||
[('length', b.length), ('blob_num', b.blob_num), ('blob_hash', b.blob_hash), ('iv', b.iv)]
|
||||
))
|
||||
if not b.blob_hash:
|
||||
break
|
||||
return json.dumps(
|
||||
OrderedDict([
|
||||
('stream_name', binascii.hexlify(self.stream_name.encode()).decode()),
|
||||
('blobs', blobs),
|
||||
('stream_type', 'lbryfile'),
|
||||
('key', self.key),
|
||||
('suggested_file_name', binascii.hexlify(self.suggested_file_name.encode()).decode()),
|
||||
('stream_hash', self.stream_hash),
|
||||
])
|
||||
).encode()
|
||||
|
||||
def calculate_old_sort_sd_hash(self):
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
|
||||
old_sort: typing.Optional[bool] = False):
|
||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||
if not old_sort:
|
||||
sd_data = self.as_json()
|
||||
else:
|
||||
sd_data = self.old_sort_json()
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
|
||||
if blob_file_obj:
|
||||
blob_file_obj.set_length(len(sd_data))
|
||||
async def make_sd_blob(self):
|
||||
sd_hash = self.calculate_sd_hash()
|
||||
sd_data = self.as_json()
|
||||
sd_blob = BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
|
||||
if not sd_blob.get_is_verified():
|
||||
writer = sd_blob.open_for_writing()
|
||||
writer.write(sd_data)
|
||||
await sd_blob.verified.wait()
|
||||
sd_blob.close()
|
||||
await sd_blob.close()
|
||||
return sd_blob
|
||||
|
||||
@classmethod
|
||||
@@ -138,8 +102,6 @@ class StreamDescriptor:
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in decoded['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
|
||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||
descriptor = cls(
|
||||
loop, blob_dir,
|
||||
binascii.unhexlify(decoded['stream_name']).decode(),
|
||||
@@ -192,8 +154,8 @@ class StreamDescriptor:
|
||||
@classmethod
|
||||
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
|
||||
old_sort: bool = False) -> 'StreamDescriptor':
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None
|
||||
) -> 'StreamDescriptor':
|
||||
|
||||
blobs: typing.List[BlobInfo] = []
|
||||
|
||||
@@ -212,7 +174,7 @@ class StreamDescriptor:
|
||||
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
|
||||
blobs
|
||||
)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
|
||||
sd_blob = await descriptor.make_sd_blob()
|
||||
descriptor.sd_hash = sd_blob.blob_hash
|
||||
return descriptor
|
||||
|
||||
@@ -222,19 +184,3 @@ class StreamDescriptor:
|
||||
|
||||
def upper_bound_decrypted_length(self) -> int:
|
||||
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
|
||||
|
||||
@classmethod
|
||||
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str,
|
||||
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
|
||||
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
|
||||
blobs, stream_hash, sd_blob.blob_hash)
|
||||
|
||||
if descriptor.calculate_sd_hash() == sd_blob.blob_hash: # first check for a normal valid sd
|
||||
old_sort = False
|
||||
elif descriptor.calculate_old_sort_sd_hash() == sd_blob.blob_hash: # check if old field sorting works
|
||||
old_sort = True
|
||||
else:
|
||||
return
|
||||
await descriptor.make_sd_blob(sd_blob, old_sort)
|
||||
return descriptor
|
||||
+209
-70
@@ -1,14 +1,13 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.utils import drain_tasks, cancel_task
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
@@ -20,77 +19,217 @@ def drain_into(a: list, b: list):
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class StreamDownloader(StreamAssembler):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_manager, sd_hash, output_file_name)
|
||||
self.config = config
|
||||
self.output_dir = output_dir or self.config.download_dir
|
||||
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
|
||||
output_file_name: typing.Optional[str] = None,
|
||||
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
max_connections_per_stream: typing.Optional[int] = 8):
|
||||
super().__init__(loop, blob_manager, sd_hash)
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.current_blob: 'BlobFile' = None
|
||||
|
||||
self.download_task: asyncio.Task = None
|
||||
self.accumulate_connections_task: asyncio.Task = None
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.output_dir = output_dir or os.getcwd()
|
||||
self.output_file_name = output_file_name
|
||||
self.blob_downloader: typing.Optional[BlobDownloader] = None
|
||||
self.search_queue = asyncio.Queue(loop=loop)
|
||||
self.peer_queue = asyncio.Queue(loop=loop)
|
||||
self.accumulate_task: typing.Optional[asyncio.Task] = None
|
||||
self.descriptor: typing.Optional[StreamDescriptor]
|
||||
self.node: typing.Optional['Node'] = None
|
||||
self.assemble_task: typing.Optional[asyncio.Task] = None
|
||||
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
|
||||
self._lock = asyncio.Lock(loop=self.loop)
|
||||
self.max_connections_per_stream = max_connections_per_stream
|
||||
self.fixed_peers = fixed_peers or []
|
||||
|
||||
async def setup(self): # start the peer accumulator and initialize the downloader
|
||||
if self.blob_downloader:
|
||||
raise Exception("downloader is already set up")
|
||||
if self.node:
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
|
||||
self.search_queue.put_nowait(self.sd_hash)
|
||||
async def _update_current_blob(self, blob: 'BlobFile'):
|
||||
async with self._lock:
|
||||
drain_tasks(self.running_download_requests)
|
||||
self.current_blob = blob
|
||||
if not blob.get_is_verified():
|
||||
self._update_requests()
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to search")
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.current_blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
|
||||
peer.address, peer.tcp_port, self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self._lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
async def after_finished(self):
|
||||
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
|
||||
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
|
||||
self.blob_downloader.close()
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.current_blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.current_blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.current_blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def stop(self):
|
||||
if self.accumulate_task:
|
||||
self.accumulate_task.cancel()
|
||||
self.accumulate_task = None
|
||||
if self.assemble_task:
|
||||
self.assemble_task.cancel()
|
||||
self.assemble_task = None
|
||||
if self.fixed_peers_handle:
|
||||
self.fixed_peers_handle.cancel()
|
||||
self.fixed_peers_handle = None
|
||||
self.blob_downloader = None
|
||||
if self.stream_handle:
|
||||
if not self.stream_handle.closed:
|
||||
self.stream_handle.close()
|
||||
self.stream_handle = None
|
||||
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
|
||||
async with self._lock:
|
||||
if len(self.running_download_requests) < self.max_connections_per_stream:
|
||||
# update the running download requests
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
async with self._lock:
|
||||
if self.current_blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.current_blob
|
||||
else:
|
||||
for task in download_tasks:
|
||||
if task and not task.done():
|
||||
self.running_download_requests.append(task)
|
||||
return
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
return await self.blob_downloader.download_blob(blob_hash, length)
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
await self._update_current_blob(blob)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
|
||||
def add_fixed_peers(self):
|
||||
async def _add_fixed_peers():
|
||||
addresses = [
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
]
|
||||
delay = self.config.fixed_peer_delay if (
|
||||
'dht' not in self.config.components_to_skip
|
||||
and self.node and len(self.node.protocol.routing_table.get_peers())
|
||||
) else 0.0
|
||||
self.loop.call_later(delay, lambda:
|
||||
self.peer_queue.put_nowait([
|
||||
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
|
||||
for address, port in addresses
|
||||
]))
|
||||
if self.config.reflector_servers:
|
||||
self.loop.create_task(_add_fixed_peers())
|
||||
# the blob must be downloaded
|
||||
try:
|
||||
while not self.current_blob.get_is_verified():
|
||||
if not self.active_connections: # wait for a new connection
|
||||
await self.new_peer_event.wait()
|
||||
continue
|
||||
blob = await self.wait_for_download_or_new_peer()
|
||||
if blob:
|
||||
drain_tasks(self.running_download_requests)
|
||||
return blob
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
|
||||
def download(self, node: typing.Optional['Node'] = None):
|
||||
self.node = node
|
||||
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
|
||||
self.add_fixed_peers()
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
blob_queue = asyncio.Queue(loop=self.loop)
|
||||
blob_queue.put_nowait(self.sd_hash)
|
||||
task = asyncio.create_task(self.got_descriptor.wait())
|
||||
added_peers = asyncio.Event(loop=self.loop)
|
||||
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
|
||||
|
||||
if self.fixed_peers:
|
||||
def check_added_peers():
|
||||
if not added_peers.is_set():
|
||||
self._add_peer_protocols(self.fixed_peers)
|
||||
log.info("no dht peers for download yet, adding fixed peer")
|
||||
added_peers.set()
|
||||
|
||||
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
|
||||
|
||||
def got_descriptor(f):
|
||||
try:
|
||||
f.result()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
log.info("add head blob hash to peer search")
|
||||
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
|
||||
task.add_done_callback(got_descriptor)
|
||||
try:
|
||||
async with node.stream_peer_search_junction(blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
if not added_peers.is_set():
|
||||
added_peers.set()
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
log.info("cancelled head blob task")
|
||||
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
|
||||
add_fixed_peers_timer.cancel()
|
||||
|
||||
async def stop(self):
|
||||
cancel_task(self.accumulate_connections_task)
|
||||
self.accumulate_connections_task = None
|
||||
drain_tasks(self.running_download_requests)
|
||||
|
||||
while self.requested_from:
|
||||
_, peer_task_dict = self.requested_from.popitem()
|
||||
while peer_task_dict:
|
||||
peer, task = peer_task_dict.popitem()
|
||||
try:
|
||||
cancel_task(task)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
while self.active_connections:
|
||||
_, client = self.active_connections.popitem()
|
||||
if client:
|
||||
await client.close()
|
||||
log.info("stopped downloader")
|
||||
|
||||
async def _download(self):
|
||||
try:
|
||||
|
||||
log.info("download and decrypt stream")
|
||||
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
|
||||
log.info(
|
||||
"downloaded stream %s -> %s", self.sd_hash, self.output_path
|
||||
)
|
||||
await self.blob_manager.storage.change_file_status(
|
||||
self.descriptor.stream_hash, 'finished'
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
await self.stop()
|
||||
|
||||
def download(self, node: 'Node'):
|
||||
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.download_task = self.loop.create_task(self._download())
|
||||
@@ -2,16 +2,13 @@ import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.reflector.client import StreamReflectorClient
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.node import Node
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,26 +18,21 @@ class ManagedStream:
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
|
||||
descriptor: 'StreamDescriptor', download_directory: str, file_name: str,
|
||||
downloader: typing.Optional[StreamDownloader] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
|
||||
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.rowid = rowid
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self.file_name = file_name
|
||||
self.descriptor = descriptor
|
||||
self.downloader = downloader
|
||||
self.stream_hash = descriptor.stream_hash
|
||||
self.stream_claim_info = claim
|
||||
self._status = status
|
||||
self._store_after_finished: asyncio.Task = None
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def file_name(self):
|
||||
return self.downloader.output_file_name if self.downloader else self._file_name
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
@@ -106,21 +98,13 @@ class ManagedStream:
|
||||
def sd_hash(self):
|
||||
return self.descriptor.sd_hash
|
||||
|
||||
@property
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
@property
|
||||
def full_path(self) -> str:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name))
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = self.full_path
|
||||
if not os.path.isfile(full_path):
|
||||
full_path = os.path.join(self.download_directory, self.file_name)
|
||||
if not os.path.exists(full_path):
|
||||
full_path = None
|
||||
mime_type = guess_media_type(os.path.basename(self.file_name))
|
||||
|
||||
if self.downloader and self.downloader.written_bytes:
|
||||
if self.downloader:
|
||||
written_bytes = self.downloader.written_bytes
|
||||
elif full_path:
|
||||
written_bytes = os.stat(full_path).st_size
|
||||
@@ -144,7 +128,6 @@ class ManagedStream:
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
'status': self.status,
|
||||
'claim_id': self.claim_id,
|
||||
'txid': self.txid,
|
||||
@@ -164,60 +147,50 @@ class ManagedStream:
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
|
||||
)
|
||||
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
|
||||
await blob_manager.blob_completed(sd_blob)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
await blob_manager.blob_completed(sd_blob)
|
||||
for blob in descriptor.blobs[:-1]:
|
||||
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
status=cls.STATUS_FINISHED)
|
||||
|
||||
def start_download(self, node: typing.Optional['Node']):
|
||||
self.downloader.download(node)
|
||||
self.update_status(self.STATUS_RUNNING)
|
||||
|
||||
def stop_download(self):
|
||||
async def stop_download(self):
|
||||
if self.downloader:
|
||||
self.downloader.stop()
|
||||
self.downloader = None
|
||||
await self.downloader.stop()
|
||||
if not self.finished:
|
||||
self.update_status(self.STATUS_STOPPED)
|
||||
|
||||
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
|
||||
sent = []
|
||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||
try:
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
except ConnectionRefusedError:
|
||||
return sent
|
||||
try:
|
||||
await protocol.send_handshake()
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
return sent
|
||||
try:
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd:
|
||||
sent.append(self.sd_hash)
|
||||
if not sent_sd and not needed:
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return []
|
||||
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
|
||||
for blob_hash in we_have:
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
return sent
|
||||
for blob_hash in needed:
|
||||
try:
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
return sent
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'ClaimDict'):
|
||||
self.stream_claim_info = StoredStreamClaim(
|
||||
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.serialized).decode(), claim.certificate_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
@@ -15,7 +15,6 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.blob_manager = blob_manager
|
||||
self.descriptor = descriptor
|
||||
@@ -27,14 +26,12 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
log.debug("Connected to reflector")
|
||||
log.info("Connected to reflector")
|
||||
self.connected.set()
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
self.transport = None
|
||||
self.connected.clear()
|
||||
if self.reflected_blobs:
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
def data_received(self, data):
|
||||
try:
|
||||
@@ -48,7 +45,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
|
||||
return await self.pending_request
|
||||
finally:
|
||||
self.pending_request = None
|
||||
@@ -83,11 +80,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
|
||||
needed.append(blob.blob_hash)
|
||||
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
|
||||
self.reflected_blobs.append(sd_blob.blob_hash)
|
||||
else:
|
||||
log.warning("Reflector failed to receive descriptor %s", sd_blob.blob_hash[:8])
|
||||
if needed:
|
||||
log.info("Reflector needs %i blobs for %s", len(needed), sd_blob.blob_hash[:8])
|
||||
return sent_sd, needed
|
||||
|
||||
async def send_blob(self, blob_hash: str):
|
||||
|
||||
@@ -37,8 +37,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error receiving blob: %s", err)
|
||||
self.transport.close()
|
||||
log.error("error downloading blob: %s", err)
|
||||
return
|
||||
try:
|
||||
request = json.loads(data.decode())
|
||||
@@ -68,33 +67,28 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
self.send_response({"received_sd_blob": True})
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"received_sd_blob": True})
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
self.send_response({"received_sd_blob": False})
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.transport.close()
|
||||
self.send_response({"received_sd_blob": False})
|
||||
return
|
||||
else:
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
if 'blob_hash' not in request:
|
||||
|
||||
+111
-306
@@ -4,27 +4,21 @@ import typing
|
||||
import binascii
|
||||
import logging
|
||||
import random
|
||||
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
|
||||
DownloadDataTimeout, DownloadSDTimeout
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.extras.daemon.storage import lbc_to_dewies
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
filter_fields = [
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'sd_hash',
|
||||
@@ -37,9 +31,7 @@ filter_fields = [
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
'full_status'
|
||||
]
|
||||
|
||||
comparison_operators = {
|
||||
@@ -53,227 +45,110 @@ comparison_operators = {
|
||||
|
||||
|
||||
class StreamManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node']):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
|
||||
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
|
||||
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.streams: typing.Set[ManagedStream] = set()
|
||||
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
|
||||
self.resume_downloading_task: asyncio.Task = None
|
||||
self.re_reflect_task: asyncio.Task = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.running_reflector_uploads: typing.List[asyncio.Task] = []
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
stream.set_claim(claim_info, smart_decode(claim_info['value']))
|
||||
|
||||
async def start_stream(self, stream: ManagedStream) -> bool:
|
||||
"""
|
||||
Resume or rebuild a partial or completed stream
|
||||
"""
|
||||
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
|
||||
if not stream.running and not os.path.isfile(path):
|
||||
if stream.downloader:
|
||||
stream.downloader.stop()
|
||||
stream.downloader = None
|
||||
|
||||
# the directory is gone, can happen when the folder that contains a published file is deleted
|
||||
# reset the download directory to the default and update the file name
|
||||
if not os.path.isdir(stream.download_directory):
|
||||
stream.download_directory = self.config.download_dir
|
||||
|
||||
stream.downloader = self.make_downloader(
|
||||
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
|
||||
)
|
||||
if stream.status != ManagedStream.STATUS_FINISHED:
|
||||
await self.storage.change_file_status(stream.stream_hash, 'running')
|
||||
stream.update_status('running')
|
||||
stream.start_download(self.node)
|
||||
try:
|
||||
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
|
||||
self.config.download_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await self.stop_stream(stream)
|
||||
if stream in self.streams:
|
||||
self.streams.remove(stream)
|
||||
return False
|
||||
file_name = os.path.basename(stream.downloader.output_path)
|
||||
output_dir = os.path.dirname(stream.downloader.output_path)
|
||||
await self.storage.change_file_download_dir_and_file_name(
|
||||
stream.stream_hash, output_dir, file_name
|
||||
)
|
||||
stream._file_name = file_name
|
||||
stream.download_directory = output_dir
|
||||
self.wait_for_stream_finished(stream)
|
||||
return True
|
||||
return True
|
||||
|
||||
async def stop_stream(self, stream: ManagedStream):
|
||||
stream.stop_download()
|
||||
if not stream.finished and os.path.isfile(stream.full_path):
|
||||
try:
|
||||
os.remove(stream.full_path)
|
||||
except OSError as err:
|
||||
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
|
||||
str(err))
|
||||
if stream.running:
|
||||
stream.update_status(ManagedStream.STATUS_STOPPED)
|
||||
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
|
||||
|
||||
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
|
||||
return StreamDownloader(
|
||||
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
|
||||
)
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
|
||||
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
blobs = await self.storage.get_blobs_for_stream(stream_hash)
|
||||
descriptor = await StreamDescriptor.recover(
|
||||
self.blob_manager.blob_dir, sd_blob, stream_hash, stream_name, suggested_file_name, key, blobs
|
||||
)
|
||||
if not descriptor:
|
||||
return
|
||||
to_restore.append((descriptor, sd_blob))
|
||||
|
||||
await asyncio.gather(*[
|
||||
recover_stream(
|
||||
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
|
||||
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
|
||||
) for file_info in file_infos
|
||||
])
|
||||
|
||||
if to_restore:
|
||||
await self.storage.recover_streams(to_restore, self.config.download_dir)
|
||||
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
|
||||
claim: typing.Optional['StoredStreamClaim']):
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
if not sd_blob.get_is_verified():
|
||||
return
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
|
||||
return
|
||||
if status == ManagedStream.STATUS_RUNNING:
|
||||
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
|
||||
else:
|
||||
downloader = None
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
|
||||
)
|
||||
self.streams.add(stream)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
self.fixed_peers = fixed_peers
|
||||
self.reflector_servers = reflector_servers
|
||||
|
||||
async def load_streams_from_database(self):
|
||||
to_recover = []
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
|
||||
to_recover.append(file_info)
|
||||
|
||||
if to_recover:
|
||||
log.info("Attempting to recover %i streams", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
to_start = []
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
|
||||
to_start.append(file_info)
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
|
||||
await asyncio.gather(*[
|
||||
self.add_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
|
||||
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
|
||||
file_info['claim']
|
||||
) for file_info in to_start
|
||||
])
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
infos = await self.storage.get_all_lbry_files()
|
||||
for file_info in infos:
|
||||
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
|
||||
if sd_blob.get_is_verified():
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
downloader = StreamDownloader(
|
||||
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
|
||||
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, descriptor,
|
||||
binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(),
|
||||
downloader, file_info['status'], file_info['claim']
|
||||
)
|
||||
self.streams.add(stream)
|
||||
|
||||
async def resume(self):
|
||||
if self.node:
|
||||
await self.node.joined.wait()
|
||||
else:
|
||||
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
t = [
|
||||
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
|
||||
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
|
||||
]
|
||||
if t:
|
||||
log.info("resuming %i downloads", len(t))
|
||||
if not self.node:
|
||||
log.warning("no DHT node given, cannot resume downloads")
|
||||
return
|
||||
await self.node.joined.wait()
|
||||
resumed = 0
|
||||
for stream in self.streams:
|
||||
if stream.status == ManagedStream.STATUS_RUNNING:
|
||||
resumed += 1
|
||||
stream.downloader.download(self.node)
|
||||
self.wait_for_stream_finished(stream)
|
||||
if resumed:
|
||||
log.info("resuming %i downloads", resumed)
|
||||
|
||||
async def reflect_streams(self):
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
|
||||
streams = list(self.streams)
|
||||
batch = []
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= 10:
|
||||
await asyncio.gather(*batch)
|
||||
batch = []
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
await asyncio.gather(*batch)
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
|
||||
async def start(self):
|
||||
await self.load_streams_from_database()
|
||||
self.resume_downloading_task = self.loop.create_task(self.resume())
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.resume_downloading_task and not self.resume_downloading_task.done():
|
||||
self.resume_downloading_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
stream = self.streams.pop()
|
||||
stream.stop_download()
|
||||
await stream.stop_download()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
self.running_reflector_uploads.pop().cancel()
|
||||
|
||||
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams.add(stream)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
task = self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
self.running_reflector_uploads.append(task)
|
||||
task.add_done_callback(
|
||||
lambda _: None
|
||||
if task not in self.running_reflector_uploads else self.running_reflector_uploads.remove(task)
|
||||
)
|
||||
if self.reflector_servers:
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
|
||||
await self.stop_stream(stream)
|
||||
if stream in self.streams:
|
||||
self.streams.remove(stream)
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await stream.stop_download()
|
||||
self.streams.remove(stream)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
if delete_file and os.path.isfile(stream.full_path):
|
||||
os.remove(stream.full_path)
|
||||
|
||||
blob_hashes = [stream.sd_hash]
|
||||
for blob_info in stream.descriptor.blobs[:-1]:
|
||||
blob_hashes.append(blob_info.blob_hash)
|
||||
for blob_hash in blob_hashes:
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
if blob.get_is_verified():
|
||||
await blob.delete()
|
||||
|
||||
if delete_file:
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
if os.path.isfile(path):
|
||||
os.remove(path)
|
||||
|
||||
def wait_for_stream_finished(self, stream: ManagedStream):
|
||||
async def _wait_for_stream_finished():
|
||||
@@ -291,84 +166,84 @@ class StreamManager:
|
||||
)
|
||||
|
||||
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
|
||||
sd_blob_timeout: typing.Optional[float] = 60
|
||||
) -> typing.Optional[ManagedStream]:
|
||||
|
||||
claim = smart_decode(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
|
||||
download_directory, file_name)
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
|
||||
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
|
||||
try:
|
||||
downloader.download(node)
|
||||
await downloader.got_descriptor.wait()
|
||||
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
|
||||
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
log.info("stream timeout")
|
||||
downloader.stop()
|
||||
await downloader.stop()
|
||||
log.info("stopped stream")
|
||||
raise DownloadSDTimeout(downloader.sd_hash)
|
||||
file_name = os.path.basename(downloader.output_path)
|
||||
download_directory = os.path.dirname(downloader.output_path)
|
||||
return
|
||||
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
|
||||
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
|
||||
rowid = await self.blob_manager.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, file_name, download_directory,
|
||||
0.0
|
||||
await self.blob_manager.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
|
||||
data_rate
|
||||
)
|
||||
else:
|
||||
rowid = self.blob_manager.storage.rowid_for_stream(downloader.descriptor.stream_hash)
|
||||
await self.blob_manager.storage.save_content_claim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, download_directory,
|
||||
file_name, downloader, ManagedStream.STATUS_RUNNING)
|
||||
stream.set_claim(claim_info, claim)
|
||||
|
||||
stored_claim = StoredStreamClaim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
|
||||
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
|
||||
claim_info.get('channel_name')
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
|
||||
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
|
||||
stored_claim)
|
||||
self.streams.add(stream)
|
||||
try:
|
||||
await stream.downloader.wrote_bytes_event.wait()
|
||||
self.wait_for_stream_finished(stream)
|
||||
return stream
|
||||
except asyncio.CancelledError:
|
||||
downloader.stop()
|
||||
log.debug("stopped stream")
|
||||
await self.stop_stream(stream)
|
||||
raise DownloadDataTimeout(downloader.sd_hash)
|
||||
await downloader.stop()
|
||||
|
||||
async def download_stream_from_claim(self, node: 'Node', claim_info: typing.Dict,
|
||||
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None,
|
||||
timeout: typing.Optional[float] = 60,
|
||||
sd_blob_timeout: typing.Optional[float] = 60,
|
||||
fee_amount: typing.Optional[float] = 0.0,
|
||||
fee_address: typing.Optional[str] = None,
|
||||
should_pay: typing.Optional[bool] = True) -> typing.Optional[ManagedStream]:
|
||||
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
if fee_address and fee_amount:
|
||||
if fee_amount > await self.wallet.default_account.get_balance():
|
||||
raise Exception("not enough funds")
|
||||
sd_hash = claim.source_hash.decode()
|
||||
if sd_hash in self.starting_streams:
|
||||
return await self.starting_streams[sd_hash]
|
||||
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
|
||||
if already_started:
|
||||
return already_started[0]
|
||||
|
||||
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
|
||||
stream_task = self.loop.create_task(
|
||||
self._download_stream_from_claim(node, self.config.download_dir, claim_info, file_name)
|
||||
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(stream_task, timeout or self.config.download_timeout)
|
||||
await asyncio.wait_for(stream_task, sd_blob_timeout)
|
||||
stream = await stream_task
|
||||
self.starting_streams[sd_hash].set_result(stream)
|
||||
if should_pay and fee_address and fee_amount:
|
||||
if fee_address and fee_amount:
|
||||
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
|
||||
return stream
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError) as e:
|
||||
if stream_task.exception():
|
||||
raise stream_task.exception()
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
return
|
||||
finally:
|
||||
if sd_hash in self.starting_streams:
|
||||
del self.starting_streams[sd_hash]
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
|
||||
if streams:
|
||||
return streams[0]
|
||||
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
|
||||
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
|
||||
comparison: typing.Optional[str] = None,
|
||||
@@ -385,8 +260,6 @@ class StreamManager:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in comparison_operators:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
for search in search_by.keys():
|
||||
if search not in filter_fields:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
@@ -395,6 +268,8 @@ class StreamManager:
|
||||
streams = []
|
||||
for stream in self.streams:
|
||||
for search, val in search_by.items():
|
||||
if search == 'full_status':
|
||||
continue
|
||||
if comparison_operators[comparison](getattr(stream, search), val):
|
||||
streams.append(stream)
|
||||
break
|
||||
@@ -405,73 +280,3 @@ class StreamManager:
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
file_name: typing.Optional[str] = None,
|
||||
timeout: typing.Optional[float] = None) -> typing.Optional[ManagedStream]:
|
||||
timeout = timeout or self.config.download_timeout
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
)
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", self.config.max_key_fee['amount']
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
else:
|
||||
balance = await self.wallet.default_account.get_balance()
|
||||
if fee_amount > balance:
|
||||
msg = f"fee of {fee_amount} exceeds max available balance"
|
||||
log.warning(msg)
|
||||
raise InsufficientFundsError(msg)
|
||||
fee_address = claim.source_fee.address.decode()
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
|
||||
if not existing:
|
||||
existing.extend(self.get_filtered_streams(sd_hash=claim.source_hash.decode()))
|
||||
if existing and existing[0].claim_id != resolved['claim_id']:
|
||||
raise Exception(f"stream for {existing[0].claim_id} collides with existing "
|
||||
f"download {resolved['claim_id']}")
|
||||
elif not existing:
|
||||
existing.extend(self.get_filtered_streams(claim_id=resolved['claim_id']))
|
||||
if existing and existing[0].sd_hash != claim.source_hash.decode():
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
stream = await self.download_stream_from_claim(
|
||||
self.node, resolved, file_name, timeout, fee_amount, fee_address, False
|
||||
)
|
||||
log.info("started new stream, deleting old one")
|
||||
await self.delete_stream(existing[0])
|
||||
return stream
|
||||
elif existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
await self.start_stream(stream)
|
||||
return stream
|
||||
else:
|
||||
stream = existing[0]
|
||||
await self.start_stream(stream)
|
||||
return stream
|
||||
log.info("download stream from %s", uri)
|
||||
return await self.download_stream_from_claim(
|
||||
self.node, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
+7
-27
@@ -8,7 +8,6 @@ import json
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
import pkg_resources
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
@@ -69,12 +68,13 @@ def obfuscate(plain):
|
||||
return rot13(base64.b64encode(plain).decode())
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
def check_connection(server="lbry.io", port=80, timeout=5):
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
try:
|
||||
server = socket.gethostbyname(server)
|
||||
socket.create_connection((server, port), timeout).close()
|
||||
conn = socket.create_connection((server, port), timeout)
|
||||
conn.close()
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except (socket.gaierror, socket.herror) as ex:
|
||||
@@ -83,23 +83,19 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
try:
|
||||
server = "8.8.8.8"
|
||||
port = 53
|
||||
socket.create_connection((server, port), timeout).close()
|
||||
socket.create_connection((server, port), timeout)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception:
|
||||
except Exception as ex:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
return False
|
||||
except Exception:
|
||||
except Exception as ex:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
server, port)
|
||||
return False
|
||||
|
||||
|
||||
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
@@ -140,19 +136,3 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
except ValueError:
|
||||
pass
|
||||
loop = asyncio.get_running_loop()
|
||||
return (await loop.getaddrinfo(
|
||||
url, port,
|
||||
proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP,
|
||||
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM
|
||||
))[0][4][0]
|
||||
@@ -1,53 +0,0 @@
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import socket
|
||||
import ipaddress
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("lbrynet")
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
async def main(blob_hash: str, url: str):
|
||||
conf = Config()
|
||||
loop = asyncio.get_running_loop()
|
||||
host_url, port = url.split(":")
|
||||
try:
|
||||
host = None
|
||||
if ipaddress.ip_address(host_url):
|
||||
host = host_url
|
||||
except ValueError:
|
||||
host = None
|
||||
if not host:
|
||||
host_info = await loop.getaddrinfo(
|
||||
host_url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
host = host_info[0][4][0]
|
||||
|
||||
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
|
||||
blob = blob_manager.get_blob(blob_hash)
|
||||
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
|
||||
conf.blob_download_timeout)
|
||||
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
|
||||
f"keep connection: {keep}")
|
||||
if blob.get_is_verified():
|
||||
await blob_manager.delete_blobs([blob.blob_hash])
|
||||
print(f"deleted {blob_hash}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
url = 'reflector.lbry.io:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
asyncio.run(main(sys.argv[1], url))
|
||||
+19
-49
@@ -19,8 +19,7 @@ except ImportError:
|
||||
|
||||
|
||||
AREA_RENAME = {
|
||||
'api': 'API',
|
||||
'dht': 'DHT'
|
||||
'api': 'API'
|
||||
}
|
||||
|
||||
|
||||
@@ -30,20 +29,16 @@ def get_github():
|
||||
with open(config_path, 'r') as config_file:
|
||||
config = json.load(config_file)
|
||||
return github3.login(token=config['token'])
|
||||
|
||||
token = os.environ.get("GH_TOKEN")
|
||||
if not token:
|
||||
print('GitHub Credentials')
|
||||
username = input('username: ')
|
||||
password = getpass('password: ')
|
||||
gh = github3.authorize(
|
||||
username, password, ['repo'], 'lbry release tool',
|
||||
two_factor_callback=lambda: input('Enter 2FA: ')
|
||||
)
|
||||
with open(config_path, 'w') as config_file:
|
||||
json.dump({'token': gh.token}, config_file)
|
||||
token = gh.token
|
||||
return github3.login(token=token)
|
||||
print('GitHub Credentials')
|
||||
username = input('username: ')
|
||||
password = getpass('password: ')
|
||||
gh = github3.authorize(
|
||||
username, password, ['repo'], 'lbry release tool',
|
||||
two_factor_callback=lambda: input('Enter 2FA: ')
|
||||
)
|
||||
with open(config_path, 'w') as config_file:
|
||||
json.dump({'token': gh.token}, config_file)
|
||||
return github3.login(token=gh.token)
|
||||
|
||||
|
||||
def get_labels(pr, prefix):
|
||||
@@ -59,7 +54,6 @@ def get_label(pr, prefix):
|
||||
|
||||
|
||||
BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:'
|
||||
RELEASE_TEXT = "release-text:"
|
||||
|
||||
|
||||
def get_backwards_incompatible(desc: str):
|
||||
@@ -68,12 +62,6 @@ def get_backwards_incompatible(desc: str):
|
||||
yield line[len(BACKWARDS_INCOMPATIBLE):]
|
||||
|
||||
|
||||
def get_release_text(desc: str):
|
||||
for line in desc.splitlines():
|
||||
if line.startswith(RELEASE_TEXT):
|
||||
yield line[len(RELEASE_TEXT):]
|
||||
|
||||
|
||||
def get_previous_final(repo, current_release):
|
||||
assert current_release.rc is not None, "Need an rc to find the previous final release."
|
||||
previous = None
|
||||
@@ -151,23 +139,15 @@ def release(args):
|
||||
previous_release = repo.release_from_tag(current_version.tag)
|
||||
|
||||
incompats = []
|
||||
release_texts = []
|
||||
unlabeled = []
|
||||
areas = {}
|
||||
for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry"):
|
||||
area_labels = list(get_labels(pr, 'area'))
|
||||
type_label = get_label(pr, 'type')
|
||||
if area_labels and type_label:
|
||||
for area_name in area_labels:
|
||||
for incompat in get_backwards_incompatible(pr.body):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
for release_text in get_release_text(pr.body):
|
||||
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
|
||||
if not (args.action == '*-rc' and type_label == 'fixup'):
|
||||
area = areas.setdefault(area_name, [])
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
else:
|
||||
unlabeled.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
for area_name in get_labels(pr, 'area'):
|
||||
area = areas.setdefault(area_name, [])
|
||||
type_label = get_label(pr, "type")
|
||||
for incompat in get_backwards_incompatible(pr.body):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
if not (args.action == '*-rc' and type_label == 'fixup'):
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
|
||||
area_names = list(areas.keys())
|
||||
area_names.sort()
|
||||
@@ -176,11 +156,6 @@ def release(args):
|
||||
w = lambda s: body.write(s+'\n')
|
||||
|
||||
w(f'## [{new_version}] - {date.today().isoformat()}')
|
||||
if release_texts:
|
||||
w('')
|
||||
for release_text in release_texts:
|
||||
w(release_text)
|
||||
w('')
|
||||
if incompats:
|
||||
w('')
|
||||
w(f'### Backwards Incompatible Changes')
|
||||
@@ -188,7 +163,7 @@ def release(args):
|
||||
w(incompat)
|
||||
for area in area_names:
|
||||
prs = areas[area]
|
||||
area = AREA_RENAME.get(area.lower(), area.capitalize())
|
||||
area = AREA_RENAME.get(area, area.capitalize())
|
||||
w('')
|
||||
w(f'### {area}')
|
||||
for pr in prs:
|
||||
@@ -196,11 +171,6 @@ def release(args):
|
||||
|
||||
print(body.getvalue())
|
||||
|
||||
if unlabeled:
|
||||
print('The following PRs were skipped and not included in changelog:')
|
||||
for skipped in unlabeled:
|
||||
print(skipped)
|
||||
|
||||
if not args.dry_run:
|
||||
|
||||
commit = version_file.update(
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
import os
|
||||
import binascii
|
||||
import sqlite3
|
||||
from lbrynet.conf import Config
|
||||
|
||||
|
||||
def main():
|
||||
conf = Config()
|
||||
db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite'))
|
||||
cur = db.cursor()
|
||||
files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall()
|
||||
update = {}
|
||||
for stream_hash, file_name, download_directory in files:
|
||||
try:
|
||||
binascii.unhexlify(file_name)
|
||||
except binascii.Error:
|
||||
try:
|
||||
binascii.unhexlify(download_directory)
|
||||
except binascii.Error:
|
||||
update[stream_hash] = (
|
||||
binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode()
|
||||
)
|
||||
if update:
|
||||
print(f"repair {len(update)} streams")
|
||||
for stream_hash, (file_name, download_directory) in update.items():
|
||||
cur.execute('update file set file_name=?, download_directory=? where stream_hash=?',
|
||||
(file_name, download_directory, stream_hash))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -7,7 +7,6 @@ import time
|
||||
|
||||
from aiohttp import ClientConnectorError
|
||||
from lbrynet import __version__
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.daemon.client import daemon_rpc
|
||||
@@ -58,29 +57,31 @@ def variance(times):
|
||||
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
|
||||
|
||||
|
||||
async def wait_for_done(conf, uri):
|
||||
async def wait_for_done(api, uri):
|
||||
name = uri.split("#")[0]
|
||||
last_complete = 0
|
||||
hang_count = 0
|
||||
while True:
|
||||
files = await daemon_rpc(conf, "file_list", claim_name=name)
|
||||
files = await api.file_list(claim_name=name)
|
||||
file = files[0]
|
||||
if file['status'] in ['finished', 'stopped']:
|
||||
return True, file['blobs_completed'], file['blobs_in_stream']
|
||||
return True, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
if last_complete < int(file['blobs_completed']):
|
||||
print(f"{file['blobs_completed']}/{file['blobs_in_stream']}...")
|
||||
hang_count = 0
|
||||
last_complete = int(file['blobs_completed'])
|
||||
else:
|
||||
hang_count += 1
|
||||
await asyncio.sleep(1.0)
|
||||
if hang_count > 10:
|
||||
return False, file['blobs_completed'], file['blobs_in_stream']
|
||||
if hang_count > 30:
|
||||
return False, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
|
||||
|
||||
async def main(uris=None):
|
||||
async def main(start_daemon=True, uris=None):
|
||||
if not uris:
|
||||
uris = await get_frontpage_uris()
|
||||
conf = Config()
|
||||
daemon = None
|
||||
try:
|
||||
await daemon_rpc(conf, 'status')
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
@@ -91,7 +92,7 @@ async def main(uris=None):
|
||||
|
||||
resolvable = []
|
||||
for name in uris:
|
||||
resolved = await daemon_rpc(conf, 'resolve', name)
|
||||
resolved = await daemon_rpc(conf, 'resolve', uri=name)
|
||||
if 'error' not in resolved.get(name, {}):
|
||||
resolvable.append(name)
|
||||
|
||||
@@ -112,24 +113,21 @@ async def main(uris=None):
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
|
||||
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
|
||||
if downloaded:
|
||||
downloaded_times.append((time.time() - start) / downloaded)
|
||||
else:
|
||||
download_failures.append(uri)
|
||||
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
|
||||
f"{round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)}mb/s\n")
|
||||
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
|
||||
# if downloaded:
|
||||
# downloaded_times.append((time.time()-start) / downloaded)
|
||||
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
|
||||
# else:
|
||||
# print(f"failed to downlload {uri}, got {msg}")
|
||||
# download_failures.append(uri)
|
||||
except:
|
||||
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
|
||||
print(f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}")
|
||||
failures.append(uri)
|
||||
return
|
||||
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
print("**********************************************")
|
||||
result = f"Tried to start downloading {len(resolvable)} streams from the front page\n" \
|
||||
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
|
||||
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
|
||||
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}\n" \
|
||||
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}\n" \
|
||||
f"Variance: {variance(first_byte_times)}\n" \
|
||||
@@ -138,7 +136,8 @@ async def main(uris=None):
|
||||
nt = '\n\t'
|
||||
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
|
||||
print(result)
|
||||
|
||||
if daemon:
|
||||
await daemon.shutdown()
|
||||
# webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
|
||||
# if webhook:
|
||||
# await report_to_slack(result, webhook)
|
||||
|
||||
Whitespace-only changes.
File renamed without changes.
@@ -1,56 +0,0 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
|
||||
async def test_account_list_set_create_remove_add(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# change account name and gap
|
||||
account_id = response['lbc_regtest'][0]['id']
|
||||
self.daemon.jsonrpc_account_set(
|
||||
account_id=account_id, new_name='test account',
|
||||
receiving_gap=95, receiving_max_uses=96,
|
||||
change_gap=97, change_max_uses=98
|
||||
)
|
||||
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
|
||||
self.assertEqual(response['name'], 'test account')
|
||||
self.assertEqual(
|
||||
response['address_generator']['receiving'],
|
||||
{'gap': 95, 'maximum_uses_per_address': 96}
|
||||
)
|
||||
self.assertEqual(
|
||||
response['address_generator']['change'],
|
||||
{'gap': 97, 'maximum_uses_per_address': 98}
|
||||
)
|
||||
|
||||
# create another account
|
||||
await self.daemon.jsonrpc_account_create('second account')
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
|
||||
account_id2 = response['lbc_regtest'][1]['id']
|
||||
|
||||
# make new account the default
|
||||
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
|
||||
response = await self.daemon.jsonrpc_account_list(show_seed=True)
|
||||
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
|
||||
|
||||
account_seed = response['lbc_regtest'][1]['seed']
|
||||
|
||||
# remove account
|
||||
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# add account
|
||||
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
|
||||
|
||||
# list specific account
|
||||
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
|
||||
self.assertEqual(response['name'], 'recreated account')
|
||||
@@ -1,224 +0,0 @@
|
||||
import tempfile
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
async def test_no_this_is_not_a_test_its_an_adventure(self):
|
||||
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
|
||||
# and everything in between; it's also the setting of some record setting
|
||||
# integration tests.
|
||||
|
||||
# Chris45 starts everyday by checking his balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '10.0')
|
||||
# "10 LBC, yippy! I can do a lot with that.", he thinks to himself,
|
||||
# enthusiastically. But he is hungry so he goes into the kitchen
|
||||
# to make himself a spamdwich.
|
||||
|
||||
# While making the spamdwich he wonders... has anyone on LBRY
|
||||
# registered the @spam channel yet? "I should do that!" he
|
||||
# exclaims and goes back to his computer to do just that!
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Do we have it locally?
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list())
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
|
||||
# As the new channel claim travels through the intertubes and makes its
|
||||
# way into the mempool and then a block and then into the claimtrie,
|
||||
# Chris doesn't sit idly by: he checks his balance!
|
||||
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# He waits for 6 more blocks (confirmations) to make sure the balance has been settled.
|
||||
await self.generate(6)
|
||||
result = await self.daemon.jsonrpc_account_balance(confirmations=6)
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# And is the channel resolvable and empty?
|
||||
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam'))
|
||||
self.assertIn('lbry://@spam', response)
|
||||
self.assertIn('certificate', response['lbry://@spam'])
|
||||
|
||||
# "What goes well with spam?" ponders Chris...
|
||||
# "A hovercraft with eels!" he exclaims.
|
||||
# "That's what goes great with spam!" he further confirms.
|
||||
|
||||
# And so, many hours later, Chris is finished writing his epic story
|
||||
# about eels driving a hovercraft across the wetlands while eating spam
|
||||
# and decides it's time to publish it to the @spam channel.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[insert long story about eels driving hovercraft]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.write(b'the end')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# He quickly checks the unconfirmed balance to make sure everything looks
|
||||
# correct.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# Also checks that his new story can be found on the blockchain before
|
||||
# giving the link to all his friends.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
|
||||
self.assertIn('lbry://@spam/hovercraft', response)
|
||||
self.assertIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
|
||||
await self.generate(5)
|
||||
# When he comes back he verifies the confirmed balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# As people start reading his story they discover some typos and notify
|
||||
# Chris who explains in despair "Oh! Noooooos!" but then remembers
|
||||
# "No big deal! I can update my claim." And so he updates his claim.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[typo fixing sounds being made]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.flush()
|
||||
claim2 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim2['success'])
|
||||
self.assertEqual(claim2['claim_id'], claim1['claim_id'])
|
||||
await self.confirm_tx(claim2['tx']['txid'])
|
||||
|
||||
# After some soul searching Chris decides that his story needs more
|
||||
# heart and a better ending. He takes down the story and begins the rewrite.
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim1['claim_id'], blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# And now checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
|
||||
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# After abandoning he just waits for his LBCs to be returned to his account
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.9693585')
|
||||
|
||||
# Amidst all this Chris receives a call from his friend Ramsey
|
||||
# who says that it is of utmost urgency that Chris transfer him
|
||||
# 1 LBC to which Chris readily obliges
|
||||
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
|
||||
ramsey_account = self.daemon.get_account_or_error(ramsey_account_id)
|
||||
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send('1.0', ramsey_address))
|
||||
self.assertIn("txid", result)
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether
|
||||
# he received it or not
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
# Chris' balance was correct
|
||||
self.assertEqual(result, '7.9692345')
|
||||
|
||||
# Ramsey too assured him that he had received the 1 LBC and thanks him
|
||||
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
|
||||
self.assertEqual(result, '1.0')
|
||||
|
||||
# After Chris is done with all the "helping other people" stuff he decides that it's time to
|
||||
# write a new story and publish it to lbry. All he needed was a fresh start and he came up with:
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'Amazingly Original First Line')
|
||||
file.write(b'Super plot for the grand novel')
|
||||
file.write(b'Totally un-cliched ending')
|
||||
file.write(b'**Audience Gasps**')
|
||||
file.flush()
|
||||
claim3 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'fresh-start', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim3['success'])
|
||||
await self.confirm_tx(claim3['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He gives the link of his story to all his friends and hopes that this is the much needed break for him
|
||||
uri = 'lbry://@spam/fresh-start'
|
||||
|
||||
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
|
||||
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support(
|
||||
'fresh-start', claim3['claim_id'], '0.2', account_id=ramsey_account_id
|
||||
))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
# It obviously did! Because, blockchain baby \O/
|
||||
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
|
||||
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now he also wanted to support the original creator of the Award Winning Novel
|
||||
# So he quickly decides to send a tip to him
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim3['claim_id'], '0.3', account_id=ramsey_account_id))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And again checks if it went to the just right place
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
# Which it obviously did. Because....?????
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Seeing the ravishing success of his novel Chris adds support to his claim too
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support('fresh-start', claim3['claim_id'], '0.4'))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
# It did!
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
|
||||
# the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered
|
||||
# something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for
|
||||
# his song, seeing as his novel had smashed all the records, he was the perfect candidate!
|
||||
# .......
|
||||
# Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'The Whale amd The Bookmark')
|
||||
file.write(b'I know right? Totally a hit song')
|
||||
file.write(b'That\'s what goes around for songs these days anyways')
|
||||
file.flush()
|
||||
claim4 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hit-song', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim4['success'])
|
||||
await self.confirm_tx(claim4['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He sends the link to Ramsey, all happy and proud
|
||||
uri = 'lbry://@spam/hit-song'
|
||||
|
||||
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
|
||||
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=claim4['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# He them checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
self.assertNotIn('claim', response[uri])
|
||||
@@ -1,373 +0,0 @@
|
||||
import tempfile
|
||||
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.error import InsufficientFundsError
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class ClaimCommands(CommandTestCase):
|
||||
|
||||
async def craft_claim(self, name, amount_dewies, claim_dict, address):
|
||||
# FIXME: this is here mostly because publish has defensive code for situations that happens accidentally
|
||||
# However, it still happens... So, let's reproduce them.
|
||||
claim = ClaimDict.load_dict(claim_dict)
|
||||
address = address or (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
tx = await Transaction.claim(name, claim, amount_dewies, address, [self.account], self.account)
|
||||
await self.broadcast(tx)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
return tx
|
||||
|
||||
async def test_create_update_and_abandon_claim(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
claim = await self.make_claim(amount='2.5') # creates new claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['claim_info']), 1)
|
||||
self.assertEqual(txs[0]['confirmations'], 1)
|
||||
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
|
||||
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.020107')
|
||||
await self.assertBalance(self.account, '7.479893')
|
||||
|
||||
await self.make_claim(amount='1.0') # updates previous claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['update_info']), 1)
|
||||
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
|
||||
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.0001985')
|
||||
await self.assertBalance(self.account, '8.9796945')
|
||||
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
await self.assertBalance(self.account, '9.9795875')
|
||||
|
||||
async def test_update_claim_holding_address(self):
|
||||
other_account_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
other_account = self.daemon.get_account_or_error(other_account_id)
|
||||
other_address = await other_account.receiving.get_or_create_usable_address()
|
||||
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
# create the initial name claim
|
||||
claim = await self.make_claim()
|
||||
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 1)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 0)
|
||||
tx = await self.daemon.jsonrpc_claim_send_to_address(
|
||||
claim['claim_id'], other_address
|
||||
)
|
||||
await self.ledger.wait(tx)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 0)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 1)
|
||||
|
||||
async def test_publishing_checks_all_accounts_for_certificate(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
spam_channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', '1.0'))
|
||||
self.assertTrue(spam_channel['success'])
|
||||
await self.confirm_tx(spam_channel['tx']['txid'])
|
||||
|
||||
self.assertEqual('8.989893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
self.assertEqual('3.989769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
baz_channel = await self.out(self.daemon.jsonrpc_channel_new('@baz', '1.0', account2_id))
|
||||
self.assertTrue(baz_channel['success'])
|
||||
await self.confirm_tx(baz_channel['tx']['txid'])
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list()))
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@baz')
|
||||
|
||||
# defaults to using all accounts to lookup channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@baz'
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# uses only the specific accounts which contains the channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account2_id]
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# fails when specifying account which does not contain channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(ValueError, "Couldn't find channel with name '@baz'."):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account1_id]
|
||||
))
|
||||
|
||||
async def test_updating_claim_includes_claim_value_in_balance_check(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
await self.make_claim(amount='9.0')
|
||||
await self.assertBalance(self.account, '0.979893')
|
||||
|
||||
# update the same claim
|
||||
await self.make_claim(amount='9.0')
|
||||
await self.assertBalance(self.account, '0.9796205')
|
||||
|
||||
# update the claim a second time but use even more funds
|
||||
await self.make_claim(amount='9.97')
|
||||
await self.assertBalance(self.account, '0.009348')
|
||||
|
||||
# fails when specifying more than available
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(
|
||||
InsufficientFundsError,
|
||||
"Please lower the bid value, the maximum amount"
|
||||
" you can specify for this claim is 9.979274."
|
||||
):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '9.98', file_path=file.name
|
||||
))
|
||||
|
||||
async def test_abandoning_claim_at_loss(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
claim = await self.make_claim(amount='0.0001')
|
||||
await self.assertBalance(self.account, '9.979793')
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
await self.assertBalance(self.account, '9.97968399')
|
||||
|
||||
async def test_claim_show(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
not_a_claim = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
|
||||
)
|
||||
self.assertEqual(not_a_claim, 'claim not found')
|
||||
|
||||
async def test_claim_list(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
unsigned_claim = await self.make_claim(amount='0.0001', name='unsigned')
|
||||
self.assertTrue(claim['success'])
|
||||
|
||||
channel_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(channel_from_claim_list['claims'][0]['value'], channel['output']['value'])
|
||||
signed_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('on-channel-claim'))
|
||||
self.assertEqual(signed_claim_from_claim_list['claims'][0]['value'], claim['output']['value'])
|
||||
unsigned_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('unsigned'))
|
||||
self.assertEqual(unsigned_claim_from_claim_list['claims'][0]['value'], unsigned_claim['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
empty = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(len(empty['claims']), 0)
|
||||
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# only possible outside a channel
|
||||
response = await self.resolve('lbry://@abc/on-channel-claim')
|
||||
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
|
||||
response = await self.resolve('lbry://on-channel-claim')
|
||||
self.assertIn('claim', response['lbry://on-channel-claim'])
|
||||
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + claim['claim_id']
|
||||
response = await self.resolve(direct_uri)
|
||||
self.assertIn('claim', response[direct_uri])
|
||||
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
|
||||
|
||||
uri = 'lbry://@abc/on-channel-claim'
|
||||
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
|
||||
original_claim = await self.make_claim(amount='0.00000001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(original_claim['success'])
|
||||
# resolves normally
|
||||
response = await self.resolve(uri)
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# tamper it, invalidating the signature
|
||||
value = response[uri]['claim']['value'].copy()
|
||||
value['stream']['metadata']['author'] = 'some troll'
|
||||
address = response[uri]['claim']['address']
|
||||
await self.craft_claim('on-channel-claim', 1, value, address)
|
||||
# it resolves to the now only valid claim under the channel, ignoring the fake one
|
||||
response = await self.resolve(uri)
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
|
||||
await self.craft_claim('on-channel-claim', 1, response[uri]['claim']['value'], address)
|
||||
|
||||
# it still resolves! but to the older claim
|
||||
response = await self.resolve(uri)
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
self.assertEqual(response[uri]['claim']['txid'], original_claim['tx']['txid'])
|
||||
|
||||
async def test_claim_list_by_channel(self):
|
||||
self.maxDiff = None
|
||||
tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "0.0001"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# 4 claims per block, 3 blocks. Sorted by height (descending) then claim_id (ascending).
|
||||
claims = []
|
||||
for j in range(3):
|
||||
same_height_claims = []
|
||||
for k in range(3):
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-{k}', channel_name='@abc', confirm=False)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-4', channel_name='@abc', confirm=True)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
same_height_claims.sort(key=lambda x: int(x, 16))
|
||||
claims = same_height_claims + claims
|
||||
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=20, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims)
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=6, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims[:6])
|
||||
out_of_bounds = await self.out(self.daemon.jsonrpc_claim_list_by_channel(2, page_size=20, uri='@abc'))
|
||||
self.assertEqual(out_of_bounds['error'], 'claim 20 greater than max 12')
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('4.999876', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# create the claim we'll be tipping and supporting
|
||||
claim = await self.make_claim()
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim['claim_id'], '1.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(tip)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tip)
|
||||
|
||||
# tips don't affect balance so account1 balance is same but account2 balance went down
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('3.9998585', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the incoming tip is marked correctly as is_tip=True in account1
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['support_info']), 1)
|
||||
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs[0]['value'], '1.0')
|
||||
self.assertEqual(txs[0]['fee'], '0.0')
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs2[0]['value'], '-1.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_claim_new_support('hovercraft', claim['claim_id'], '2.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(support)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(support)
|
||||
|
||||
# account2 balance went down ~2
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('1.999717', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
|
||||
self.assertEqual(txs2[0]['value'], '0.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
@@ -1,116 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class FileCommands(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.INFO
|
||||
|
||||
async def test_file_management(self):
|
||||
await self.make_claim('foo', '0.01')
|
||||
await self.make_claim('foo2', '0.01')
|
||||
|
||||
file1, file2 = self.daemon.jsonrpc_file_list('claim_name')
|
||||
self.assertEqual(file1['claim_name'], 'foo')
|
||||
self.assertEqual(file2['claim_name'], 'foo2')
|
||||
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
|
||||
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
async def test_download_different_timeouts(self):
|
||||
claim = await self.make_claim('foo', '0.01')
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
all_except_sd = [
|
||||
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
|
||||
]
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd)
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEquals('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
await self.server.blob_manager.delete_blobs([sd_hash])
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEquals('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
|
||||
|
||||
async def wait_files_to_complete(self):
|
||||
while self.daemon.jsonrpc_file_list(status='running'):
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_filename_conflicts_management_on_resume_download(self):
|
||||
await self.make_claim('foo', '0.01', data=bytes([0]*(1<<23)))
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
with open(original_path, 'wb') as handle:
|
||||
handle.write(b'some other stuff was there instead')
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
|
||||
# check that internal state got through up to the file list API
|
||||
downloader = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']).downloader
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertEqual(downloader.output_file_name, file_info['file_name'])
|
||||
# checks if what the API shows is what he have at the very internal level.
|
||||
self.assertEqual(downloader.output_path, file_info['download_path'])
|
||||
# if you got here refactoring just change above, but ensure what gets set internally gets reflected externally!
|
||||
self.assertTrue(downloader.output_path.endswith(downloader.output_file_name))
|
||||
# this used to be inconsistent, if it becomes again it would create weird bugs, so worth checking
|
||||
|
||||
async def test_incomplete_downloads_erases_output_file_on_stop(self):
|
||||
claim = await self.make_claim('foo', '0.01')
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
all_except_sd = [
|
||||
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
|
||||
]
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd)
|
||||
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertIn('error', resp)
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
|
||||
async def test_incomplete_downloads_retry(self):
|
||||
claim = await self.make_claim('foo', '0.01')
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
all_except_sd = [
|
||||
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
|
||||
]
|
||||
|
||||
# backup server blobs
|
||||
for blob_hash in all_except_sd:
|
||||
blob = self.server_blob_manager.get_blob(blob_hash)
|
||||
os.rename(blob.file_path, blob.file_path + '__')
|
||||
|
||||
# erase all except sd blob
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd)
|
||||
|
||||
# fails, as expected
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
self.assertEqual('stopped', self.daemon.jsonrpc_file_list()[0]['status'])
|
||||
|
||||
# recover blobs
|
||||
for blob_hash in all_except_sd:
|
||||
blob = self.server_blob_manager.get_blob(blob_hash)
|
||||
os.rename(blob.file_path + '__', blob.file_path)
|
||||
self.server_blob_manager.blobs.clear()
|
||||
await self.server_blob_manager.blob_completed(self.server_blob_manager.get_blob(blob_hash))
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5)
|
||||
self.assertNotIn('error', resp)
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
|
||||
@@ -1,59 +0,0 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class ResolveCommand(CommandTestCase):
|
||||
|
||||
async def test_resolve(self):
|
||||
await self.make_channel('@abc', '0.01')
|
||||
|
||||
# resolving a channel @abc
|
||||
response = await self.resolve('lbry://@abc')
|
||||
self.assertSetEqual({'lbry://@abc'}, set(response))
|
||||
self.assertIn('certificate', response['lbry://@abc'])
|
||||
self.assertNotIn('claim', response['lbry://@abc'])
|
||||
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
|
||||
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 0)
|
||||
|
||||
await self.make_claim('foo', '0.01', channel_name='@abc')
|
||||
await self.make_claim('foo2', '0.01', channel_name='@abc')
|
||||
|
||||
# resolving a channel @abc with some claims in it
|
||||
response = await self.resolve('lbry://@abc')
|
||||
self.assertSetEqual({'lbry://@abc'}, set(response))
|
||||
self.assertIn('certificate', response['lbry://@abc'])
|
||||
self.assertNotIn('claim', response['lbry://@abc'])
|
||||
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
|
||||
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 2)
|
||||
|
||||
# resolving claim foo within channel @abc
|
||||
response = await self.resolve('lbry://@abc/foo')
|
||||
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
|
||||
claim = response['lbry://@abc/foo']
|
||||
self.assertIn('certificate', claim)
|
||||
self.assertIn('claim', claim)
|
||||
self.assertEqual(claim['claim']['name'], 'foo')
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
|
||||
# resolving claim foo by itself
|
||||
response = await self.resolve('lbry://foo')
|
||||
self.assertSetEqual({'lbry://foo'}, set(response))
|
||||
claim = response['lbry://foo']
|
||||
self.assertIn('certificate', claim)
|
||||
self.assertIn('claim', claim)
|
||||
self.assertEqual(claim['claim']['name'], 'foo')
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
|
||||
# resolving multiple at once
|
||||
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
|
||||
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
|
||||
claim = response['lbry://foo2']
|
||||
self.assertIn('certificate', claim)
|
||||
self.assertIn('claim', claim)
|
||||
self.assertEqual(claim['claim']['name'], 'foo2')
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
@@ -1,38 +0,0 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
|
||||
self.assertEqual(tx.id, result['txid'])
|
||||
|
||||
# someone's tx
|
||||
change_address = await self.blockchain.get_raw_change_address()
|
||||
sendtxid = await self.blockchain.send_to_address(change_address, 10)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.id, sendtxid)
|
||||
self.assertEqual(tx.height, -2)
|
||||
await self.generate(1)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.height, self.ledger.headers.height)
|
||||
|
||||
# inexistent
|
||||
result = await self.daemon.jsonrpc_transaction_show('0'*64)
|
||||
self.assertFalse(result['success'])
|
||||
|
||||
async def test_utxo_release(self):
|
||||
sendtxid = await self.blockchain.send_to_address(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1
|
||||
)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
await self.ledger.reserve_outputs(await self.account.get_utxos())
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
await self.daemon.jsonrpc_utxo_release()
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
@@ -1,186 +0,0 @@
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
import lbrynet.extras.wallet
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.Components import Component, WalletComponent
|
||||
from lbrynet.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.reflector.server import ReflectorServer
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
|
||||
|
||||
class ExchangeRateManager:
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
return amount
|
||||
|
||||
def fee_dict(self):
|
||||
return {}
|
||||
|
||||
|
||||
class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
def component(self) -> ExchangeRateManager:
|
||||
return self.exchange_rate_manager
|
||||
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
|
||||
|
||||
class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
timeout = 180
|
||||
LEDGER = lbrynet.extras.wallet
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
logging.getLogger('lbrynet.blob_exchange').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.stream').setLevel(self.VERBOSITY)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = self.wallet_node.data_path
|
||||
conf.wallet_dir = self.wallet_node.data_path
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = True
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
sendtxid = await self.blockchain.send_to_address(address, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(5)
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
self.wallet_component = WalletComponent(component_manager)
|
||||
self.wallet_component.wallet_manager = self.manager
|
||||
self.wallet_component._running = True
|
||||
return self.wallet_component
|
||||
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
self.daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
||||
exchange_rate_manager=ExchangeRateManagerComponent
|
||||
))
|
||||
await self.daemon.initialize()
|
||||
self.manager.old_db = self.daemon.storage
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_storage = SQLiteStorage(Config(), ':memory:')
|
||||
await self.server_storage.open()
|
||||
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
self.server.start_server(5567, '127.0.0.1')
|
||||
await self.server.started_listening.wait()
|
||||
|
||||
self.reflector = ReflectorServer(self.server_blob_manager)
|
||||
self.reflector.start_server(5566, '127.0.0.1')
|
||||
await self.reflector.started_listening.wait()
|
||||
self.addCleanup(self.reflector.stop_server)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
await self.on_transaction_id(txid)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid)
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
await self.ledger.wait(
|
||||
self.ledger.transaction_class(unhexlify(tx['hex']))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_all_addresses(tx):
|
||||
addresses = set()
|
||||
for txi in tx['inputs']:
|
||||
addresses.add(txi['address'])
|
||||
for txo in tx['outputs']:
|
||||
addresses.add(txo['address'])
|
||||
return list(addresses)
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
await self.blockchain.generate(blocks)
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def out(self, awaitable):
|
||||
""" Converts Daemon API call results (dictionary)
|
||||
to JSON and then back to a dictionary. """
|
||||
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
|
||||
|
||||
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!',
|
||||
channel_name=None, confirm=True, account_id=None):
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(self.daemon.jsonrpc_publish(
|
||||
name, amount, file_path=file.name, channel_name=channel_name, account_id=account_id
|
||||
))
|
||||
self.assertTrue(claim['success'])
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
return claim
|
||||
|
||||
async def make_channel(self, name='@arena', amount='1.0', confirm=True, account_id=None):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new(name, amount, account_id))
|
||||
self.assertTrue(channel['success'])
|
||||
if confirm:
|
||||
await self.on_transaction_dict(channel['tx'])
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(channel['tx'])
|
||||
return channel
|
||||
|
||||
async def resolve(self, uri):
|
||||
return await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
Whitespace-only changes.
@@ -0,0 +1,804 @@
|
||||
import json
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.error import InsufficientFundsError
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.Components import WalletComponent
|
||||
from lbrynet.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
|
||||
class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
timeout = 180
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
logging.getLogger('lbrynet.blob_exchange').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = self.wallet_node.data_path
|
||||
conf.wallet_dir = self.wallet_node.data_path
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_uploads = False
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('localhost', 50001)]
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
sendtxid = await self.blockchain.send_to_address(address, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(5)
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
self.wallet_component = WalletComponent(component_manager)
|
||||
self.wallet_component.wallet_manager = self.manager
|
||||
self.wallet_component._running = True
|
||||
return self.wallet_component
|
||||
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
]
|
||||
self.daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
|
||||
))
|
||||
await self.daemon.initialize()
|
||||
self.manager.old_db = self.daemon.storage
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
await self.on_transaction_id(txid)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid)
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
await self.ledger.wait(
|
||||
self.ledger.transaction_class(unhexlify(tx['hex']))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_all_addresses(tx):
|
||||
addresses = set()
|
||||
for txi in tx['inputs']:
|
||||
addresses.add(txi['address'])
|
||||
for txo in tx['outputs']:
|
||||
addresses.add(txo['address'])
|
||||
return list(addresses)
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
await self.blockchain.generate(blocks)
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def out(self, awaitable):
|
||||
""" Converts Daemon API call results (dictionary)
|
||||
to JSON and then back to a dictionary. """
|
||||
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
|
||||
|
||||
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def test_no_this_is_not_a_test_its_an_adventure(self):
|
||||
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
|
||||
# and everything in between; it's also the setting of some record setting
|
||||
# integration tests.
|
||||
|
||||
# Chris45 starts everyday by checking his balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '10.0')
|
||||
# "10 LBC, yippy! I can do a lot with that.", he thinks to himself,
|
||||
# enthusiastically. But he is hungry so he goes into the kitchen
|
||||
# to make himself a spamdwich.
|
||||
|
||||
# While making the spamdwich he wonders... has anyone on LBRY
|
||||
# registered the @spam channel yet? "I should do that!" he
|
||||
# exclaims and goes back to his computer to do just that!
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Do we have it locally?
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list())
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
|
||||
# As the new channel claim travels through the intertubes and makes its
|
||||
# way into the mempool and then a block and then into the claimtrie,
|
||||
# Chris doesn't sit idly by: he checks his balance!
|
||||
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# He waits for 6 more blocks (confirmations) to make sure the balance has been settled.
|
||||
await self.generate(6)
|
||||
result = await self.daemon.jsonrpc_account_balance(confirmations=6)
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# And is the channel resolvable and empty?
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam'))
|
||||
self.assertIn('lbry://@spam', response)
|
||||
self.assertIn('certificate', response['lbry://@spam'])
|
||||
|
||||
# "What goes well with spam?" ponders Chris...
|
||||
# "A hovercraft with eels!" he exclaims.
|
||||
# "That's what goes great with spam!" he further confirms.
|
||||
|
||||
# And so, many hours later, Chris is finished writing his epic story
|
||||
# about eels driving a hovercraft across the wetlands while eating spam
|
||||
# and decides it's time to publish it to the @spam channel.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[insert long story about eels driving hovercraft]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.write(b'the end')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# He quickly checks the unconfirmed balance to make sure everything looks
|
||||
# correct.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# Also checks that his new story can be found on the blockchain before
|
||||
# giving the link to all his friends.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam/hovercraft'))
|
||||
self.assertIn('lbry://@spam/hovercraft', response)
|
||||
self.assertIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
|
||||
await self.generate(5)
|
||||
# When he comes back he verifies the confirmed balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# As people start reading his story they discover some typos and notify
|
||||
# Chris who explains in despair "Oh! Noooooos!" but then remembers
|
||||
# "No big deal! I can update my claim." And so he updates his claim.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[typo fixing sounds being made]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.flush()
|
||||
claim2 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim2['success'])
|
||||
self.assertEqual(claim2['claim_id'], claim1['claim_id'])
|
||||
await self.confirm_tx(claim2['tx']['txid'])
|
||||
|
||||
# After some soul searching Chris decides that his story needs more
|
||||
# heart and a better ending. He takes down the story and begins the rewrite.
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim1['claim_id'], blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# And now checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam/hovercraft'))
|
||||
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# After abandoning he just waits for his LBCs to be returned to his account
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.9693585')
|
||||
|
||||
# Amidst all this Chris receives a call from his friend Ramsey
|
||||
# who says that it is of utmost urgency that Chris transfer him
|
||||
# 1 LBC to which Chris readily obliges
|
||||
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
|
||||
ramsey_account = self.daemon.get_account_or_error(ramsey_account_id)
|
||||
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send('1.0', ramsey_address))
|
||||
self.assertIn("txid", result)
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether
|
||||
# he received it or not
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
# Chris' balance was correct
|
||||
self.assertEqual(result, '7.9692345')
|
||||
|
||||
# Ramsey too assured him that he had received the 1 LBC and thanks him
|
||||
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
|
||||
self.assertEqual(result, '1.0')
|
||||
|
||||
# After Chris is done with all the "helping other people" stuff he decides that it's time to
|
||||
# write a new story and publish it to lbry. All he needed was a fresh start and he came up with:
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'Amazingly Original First Line')
|
||||
file.write(b'Super plot for the grand novel')
|
||||
file.write(b'Totally un-cliched ending')
|
||||
file.write(b'**Audience Gasps**')
|
||||
file.flush()
|
||||
claim3 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'fresh-start', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim3['success'])
|
||||
await self.confirm_tx(claim3['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He gives the link of his story to all his friends and hopes that this is the much needed break for him
|
||||
uri = 'lbry://@spam/fresh-start'
|
||||
|
||||
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
|
||||
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support(
|
||||
'fresh-start', claim3['claim_id'], '0.2', account_id=ramsey_account_id
|
||||
))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
# It obviously did! Because, blockchain baby \O/
|
||||
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
|
||||
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now he also wanted to support the original creator of the Award Winning Novel
|
||||
# So he quickly decides to send a tip to him
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim3['claim_id'], '0.3', account_id=ramsey_account_id))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And again checks if it went to the just right place
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
# Which it obviously did. Because....?????
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Seeing the ravishing success of his novel Chris adds support to his claim too
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support('fresh-start', claim3['claim_id'], '0.4'))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
# It did!
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
|
||||
# the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered
|
||||
# something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for
|
||||
# his song, seeing as his novel had smashed all the records, he was the perfect candidate!
|
||||
# .......
|
||||
# Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'The Whale amd The Bookmark')
|
||||
file.write(b'I know right? Totally a hit song')
|
||||
file.write(b'That\'s what goes around for songs these days anyways')
|
||||
file.flush()
|
||||
claim4 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hit-song', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim4['success'])
|
||||
await self.confirm_tx(claim4['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He sends the link to Ramsey, all happy and proud
|
||||
uri = 'lbry://@spam/hit-song'
|
||||
|
||||
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
|
||||
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=claim4['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# He them checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertNotIn('claim', response[uri])
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
|
||||
VERBOSE = False
|
||||
|
||||
async def test_performing_account_management_commands(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# change account name and gap
|
||||
account_id = response['lbc_regtest'][0]['id']
|
||||
self.daemon.jsonrpc_account_set(
|
||||
account_id=account_id, new_name='test account',
|
||||
receiving_gap=95, receiving_max_uses=96,
|
||||
change_gap=97, change_max_uses=98
|
||||
)
|
||||
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
|
||||
self.assertEqual(response['name'], 'test account')
|
||||
self.assertEqual(
|
||||
response['address_generator']['receiving'],
|
||||
{'gap': 95, 'maximum_uses_per_address': 96}
|
||||
)
|
||||
self.assertEqual(
|
||||
response['address_generator']['change'],
|
||||
{'gap': 97, 'maximum_uses_per_address': 98}
|
||||
)
|
||||
|
||||
# create another account
|
||||
await self.daemon.jsonrpc_account_create('second account')
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
|
||||
account_id2 = response['lbc_regtest'][1]['id']
|
||||
|
||||
# make new account the default
|
||||
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
|
||||
response = await self.daemon.jsonrpc_account_list(show_seed=True)
|
||||
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
|
||||
|
||||
account_seed = response['lbc_regtest'][1]['seed']
|
||||
|
||||
# remove account
|
||||
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# add account
|
||||
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
|
||||
|
||||
# list specific account
|
||||
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
|
||||
self.assertEqual(response['name'], 'recreated account')
|
||||
|
||||
|
||||
class ClaimManagement(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!', channel_name=None, confirm=True):
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(self.daemon.jsonrpc_publish(
|
||||
name, amount, file_path=file.name, channel_name=channel_name
|
||||
))
|
||||
self.assertTrue(claim['success'])
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
return claim
|
||||
|
||||
async def craft_claim(self, name, amount_dewies, claim_dict, address):
|
||||
# FIXME: this is here mostly because publish has defensive code for situations that happens accidentally
|
||||
# However, it still happens... So, let's reproduce them.
|
||||
claim = ClaimDict.load_dict(claim_dict)
|
||||
address = address or (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
tx = await Transaction.claim(name, claim, amount_dewies, address, [self.account], self.account)
|
||||
await self.broadcast(tx)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
return tx
|
||||
|
||||
async def test_create_update_and_abandon_claim(self):
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
claim = await self.make_claim(amount='2.5') # creates new claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['claim_info']), 1)
|
||||
self.assertEqual(txs[0]['confirmations'], 1)
|
||||
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
|
||||
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.020107')
|
||||
self.assertEqual('7.479893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
await self.make_claim(amount='1.0') # updates previous claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['update_info']), 1)
|
||||
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
|
||||
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.0001985')
|
||||
self.assertEqual('8.9796945', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
self.assertEqual('9.9795875', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
async def test_update_claim_holding_address(self):
|
||||
other_account_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
other_account = self.daemon.get_account_or_error(other_account_id)
|
||||
other_address = await other_account.receiving.get_or_create_usable_address()
|
||||
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# create the initial name claim
|
||||
claim = await self.make_claim()
|
||||
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 1)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 0)
|
||||
tx = await self.daemon.jsonrpc_claim_send_to_address(
|
||||
claim['claim_id'], other_address
|
||||
)
|
||||
await self.ledger.wait(tx)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 0)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 1)
|
||||
|
||||
async def test_publishing_checks_all_accounts_for_certificate(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
spam_channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', '1.0'))
|
||||
self.assertTrue(spam_channel['success'])
|
||||
await self.confirm_tx(spam_channel['tx']['txid'])
|
||||
|
||||
self.assertEqual('8.989893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
self.assertEqual('3.989769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
baz_channel = await self.out(self.daemon.jsonrpc_channel_new('@baz', '1.0', account2_id))
|
||||
self.assertTrue(baz_channel['success'])
|
||||
await self.confirm_tx(baz_channel['tx']['txid'])
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list()))
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@baz')
|
||||
|
||||
# defaults to using all accounts to lookup channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@baz'
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# uses only the specific accounts which contains the channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account2_id]
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# fails when specifying account which does not contain channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(ValueError, "Couldn't find channel with name '@baz'."):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account1_id]
|
||||
))
|
||||
|
||||
async def test_updating_claim_includes_claim_value_in_balance_check(self):
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
await self.make_claim(amount='9.0')
|
||||
self.assertEqual('0.979893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# update the same claim
|
||||
await self.make_claim(amount='9.0')
|
||||
self.assertEqual('0.9796205', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# update the claim a second time but use even more funds
|
||||
await self.make_claim(amount='9.97')
|
||||
self.assertEqual('0.009348', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# fails when specifying more than available
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(
|
||||
InsufficientFundsError,
|
||||
"Please lower the bid value, the maximum amount"
|
||||
" you can specify for this claim is 9.979274."
|
||||
):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '9.98', file_path=file.name
|
||||
))
|
||||
|
||||
async def test_abandoning_claim_at_loss(self):
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
claim = await self.make_claim(amount='0.0001')
|
||||
self.assertEqual('9.979793', await self.daemon.jsonrpc_account_balance())
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
self.assertEqual('9.97968399', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
async def test_claim_show(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
not_a_claim = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
|
||||
)
|
||||
self.assertEqual(not_a_claim, 'claim not found')
|
||||
|
||||
async def test_claim_list(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
unsigned_claim = await self.make_claim(amount='0.0001', name='unsigned')
|
||||
self.assertTrue(claim['success'])
|
||||
|
||||
channel_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(channel_from_claim_list['claims'][0]['value'], channel['output']['value'])
|
||||
signed_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('on-channel-claim'))
|
||||
self.assertEqual(signed_claim_from_claim_list['claims'][0]['value'], claim['output']['value'])
|
||||
unsigned_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('unsigned'))
|
||||
self.assertEqual(unsigned_claim_from_claim_list['claims'][0]['value'], unsigned_claim['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
empty = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(len(empty['claims']), 0)
|
||||
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# only possible outside a channel
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@abc/on-channel-claim'))
|
||||
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://on-channel-claim'))
|
||||
self.assertIn('claim', response['lbry://on-channel-claim'])
|
||||
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + claim['claim_id']
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=direct_uri))
|
||||
self.assertIn('claim', response[direct_uri])
|
||||
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
|
||||
|
||||
uri = 'lbry://@abc/on-channel-claim'
|
||||
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
|
||||
original_claim = await self.make_claim(amount='0.00000001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(original_claim['success'])
|
||||
# resolves normally
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# tamper it, invalidating the signature
|
||||
value = response[uri]['claim']['value'].copy()
|
||||
value['stream']['metadata']['author'] = 'some troll'
|
||||
address = response[uri]['claim']['address']
|
||||
await self.craft_claim('on-channel-claim', 1, value, address)
|
||||
# it resolves to the now only valid claim under the channel, ignoring the fake one
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
|
||||
await self.craft_claim('on-channel-claim', 1, response[uri]['claim']['value'], address)
|
||||
|
||||
# it still resolves! but to the older claim
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
self.assertEqual(response[uri]['claim']['txid'], original_claim['tx']['txid'])
|
||||
|
||||
async def test_claim_list_by_channel(self):
|
||||
self.maxDiff = None
|
||||
tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "0.0001"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# 4 claims per block, 3 blocks. Sorted by height (descending) then claim_id (ascending).
|
||||
claims = []
|
||||
for j in range(3):
|
||||
same_height_claims = []
|
||||
for k in range(3):
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-{k}', channel_name='@abc', confirm=False)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-4', channel_name='@abc', confirm=True)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
same_height_claims.sort(key=lambda x: int(x, 16))
|
||||
claims = same_height_claims + claims
|
||||
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=20, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims)
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=6, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims[:6])
|
||||
out_of_bounds = await self.out(self.daemon.jsonrpc_claim_list_by_channel(2, page_size=20, uri='@abc'))
|
||||
self.assertEqual(out_of_bounds['error'], 'claim 20 greater than max 12')
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('4.999876', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# create the claim we'll be tipping and supporting
|
||||
claim = await self.make_claim()
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim['claim_id'], '1.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(tip)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tip)
|
||||
|
||||
# tips don't affect balance so account1 balance is same but account2 balance went down
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('3.9998585', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the incoming tip is marked correctly as is_tip=True in account1
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['support_info']), 1)
|
||||
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs[0]['value'], '1.0')
|
||||
self.assertEqual(txs[0]['fee'], '0.0')
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs2[0]['value'], '-1.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_claim_new_support('hovercraft', claim['claim_id'], '2.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(support)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(support)
|
||||
|
||||
# account2 balance went down ~2
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('1.999717', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
|
||||
self.assertEqual(txs2[0]['value'], '0.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
|
||||
self.assertEqual(tx.id, result['txid'])
|
||||
|
||||
# someone's tx
|
||||
change_address = await self.blockchain.get_raw_change_address()
|
||||
sendtxid = await self.blockchain.send_to_address(change_address, 10)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.id, sendtxid)
|
||||
self.assertEqual(tx.height, -2)
|
||||
await self.generate(1)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.height, self.ledger.headers.height)
|
||||
|
||||
# inexistent
|
||||
result = await self.daemon.jsonrpc_transaction_show('0'*64)
|
||||
self.assertFalse(result['success'])
|
||||
|
||||
async def test_utxo_release(self):
|
||||
sendtxid = await self.blockchain.send_to_address(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1
|
||||
)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
await self.ledger.reserve_outputs(await self.account.get_utxos())
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
await self.daemon.jsonrpc_utxo_release()
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
+1
-2
@@ -8,7 +8,6 @@ from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from torba.testcase import IntegrationTestCase
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
import lbrynet.extras.wallet
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.extras.wallet.account import generate_certificate
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
|
||||
@@ -46,7 +45,7 @@ example_claim_dict = {
|
||||
|
||||
class BasicTransactionTest(IntegrationTestCase):
|
||||
|
||||
LEDGER = lbrynet.extras.wallet
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def test_creating_updating_and_abandoning_claim_with_channel(self):
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
|
||||
|
||||
class TestBlobManager(AsyncioTestCase):
|
||||
async def test_sync_blob_manager_on_startup(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
|
||||
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
|
||||
# add a blob file
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
with open(os.path.join(blob_manager.blob_dir, blob_hash), 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
# it should not have been added automatically on startup
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
|
||||
# make sure we can add the blob
|
||||
await blob_manager.blob_completed(blob_manager.get_blob(blob_hash, len(blob_bytes)))
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
|
||||
|
||||
# stop the blob manager and restart it, make sure the blob is there
|
||||
blob_manager.stop()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
|
||||
|
||||
# test that the blob is removed upon the next startup after the file being manually deleted
|
||||
blob_manager.stop()
|
||||
|
||||
# manually delete the blob file and restart the blob manager
|
||||
os.remove(os.path.join(blob_manager.blob_dir, blob_hash))
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
|
||||
# check that the deleted blob was updated in the database
|
||||
self.assertEqual(
|
||||
'pending', (
|
||||
await storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
|
||||
)
|
||||
)
|
||||
@@ -1,16 +1,12 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
|
||||
import shutil
|
||||
import os
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobRequest
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
@@ -18,12 +14,6 @@ from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
# logging.getLogger("lbrynet").setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def mock_config():
|
||||
config = Config()
|
||||
config.fixed_peer_delay = 10000
|
||||
return config
|
||||
|
||||
|
||||
class BlobExchangeTestBase(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
@@ -32,15 +22,12 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.server_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.client_dir)
|
||||
self.addCleanup(shutil.rmtree, self.server_dir)
|
||||
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
|
||||
self.server_storage = SQLiteStorage(Config(), os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
|
||||
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
|
||||
reflector_servers=[])
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_storage = SQLiteStorage(Config(), os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
@@ -66,11 +53,12 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
async def _test_transfer_blob(self, blob_hash: str):
|
||||
client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2, 3)
|
||||
await client_blob.finished_writing.wait()
|
||||
downloaded = await request_blob(self.loop, client_blob, protocol, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2)
|
||||
await protocol.close()
|
||||
self.assertEqual(client_blob.get_is_verified(), True)
|
||||
self.assertTrue(downloaded)
|
||||
|
||||
@@ -103,16 +91,17 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
),
|
||||
self._test_transfer_blob(blob_hash)
|
||||
)
|
||||
await second_client_blob.finished_writing.wait()
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_host_different_blobs_to_multiple_peers_at_once(self):
|
||||
@@ -136,26 +125,14 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
),
|
||||
self._test_transfer_blob(sd_hash),
|
||||
second_client_blob.finished_writing.wait()
|
||||
self._test_transfer_blob(sd_hash)
|
||||
)
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_server_chunked_request(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address)
|
||||
transport = asyncio.Transport(extra={'peername': ('ip', 90)})
|
||||
received_data = BytesIO()
|
||||
transport.write = received_data.write
|
||||
server_protocol.connection_made(transport)
|
||||
blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize()
|
||||
for byte in blob_request:
|
||||
server_protocol.data_received(bytes([byte]))
|
||||
await asyncio.sleep(0.1) # yield execution
|
||||
self.assertTrue(len(received_data.getvalue()) > 0)
|
||||
@@ -78,7 +78,8 @@ class StorageTest(AsyncioTestCase):
|
||||
await self.storage.close()
|
||||
|
||||
async def store_fake_blob(self, blob_hash, length=100):
|
||||
await self.storage.add_completed_blob(blob_hash, length)
|
||||
await self.storage.add_known_blob(blob_hash, length)
|
||||
await self.storage.add_completed_blob(blob_hash)
|
||||
|
||||
async def store_fake_stream(self, stream_hash, blobs=None, file_name="fake_file", key="DEADBEEF"):
|
||||
blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", random_lbry_hash())]
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbrynet.dht.serialization.bencoding import bencode, bdecode
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
@@ -69,7 +67,6 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
|
||||
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
|
||||
)
|
||||
peer2_from_peer1.update_tcp_port(3333)
|
||||
peer3 = peer1.peer_manager.get_kademlia_peer(
|
||||
constants.generate_id(), '1.2.3.6', udp_port=4444
|
||||
)
|
||||
@@ -79,15 +76,14 @@ class TestProtocol(AsyncioTestCase):
|
||||
self.assertEqual(True, peer1.data_store.has_peers_for_blob(b'2' * 48))
|
||||
self.assertEqual(False, peer1.data_store.has_peers_for_blob(b'3' * 48))
|
||||
self.assertListEqual([peer2_from_peer1], peer1.data_store.get_storing_contacts())
|
||||
peer1.data_store.completed_blobs.add(binascii.hexlify(b'2' * 48).decode())
|
||||
|
||||
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
|
||||
self.assertEqual(len(find_value_response[b'contacts']), 0)
|
||||
self.assertSetEqual(
|
||||
{b'2' * 48, b'token', b'protocolVersion', b'contacts'}, set(find_value_response.keys())
|
||||
{b'2' * 48, b'token', b'protocolVersion'}, set(find_value_response.keys())
|
||||
)
|
||||
self.assertEqual(2, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
|
||||
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
|
||||
self.assertEqual(1, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1)
|
||||
# self.assertEqual(peer2_from_peer1.tcp_port, 3333)
|
||||
|
||||
peer1.stop()
|
||||
peer2.stop()
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import unittest
|
||||
import asyncio
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -42,17 +41,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
|
||||
async def test_yield_order(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_one_stopped_first(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.101, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
@@ -64,16 +62,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
for i in range(10):
|
||||
if i == 5:
|
||||
return
|
||||
await asyncio.sleep(0.2)
|
||||
await asyncio.sleep(0.101)
|
||||
yield 1
|
||||
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
await self._test_junction(expected_order, fast_gen(), slow_gen)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
async def test_stop_when_encapsulating_task_cancelled(self):
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
|
||||
async def _task():
|
||||
async with AsyncGeneratorJunction(self.loop) as junction:
|
||||
@@ -83,7 +81,7 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
pass
|
||||
|
||||
task = self.loop.create_task(_task())
|
||||
self.loop.call_later(1.0, task.cancel)
|
||||
self.loop.call_later(0.5, task.cancel)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
|
||||
@@ -84,6 +84,15 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_fee_and_ungenerous_data(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
fake_fee_amount = 4.5
|
||||
correct_result = size / 10 ** 6 * conf.data_rate + fake_fee_amount
|
||||
daemon = get_test_daemon(conf, with_fee=True)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
def test_generous_data_and_no_fee(self):
|
||||
size = 10000000
|
||||
correct_result = 0.0
|
||||
@@ -91,6 +100,14 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_ungenerous_data_and_no_fee(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
correct_result = size / 10 ** 6 * conf.data_rate
|
||||
daemon = get_test_daemon(conf)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
|
||||
@unittest.SkipTest
|
||||
class TestJsonRpc(unittest.TestCase):
|
||||
|
||||
@@ -35,14 +35,6 @@ class DummyExchangeRateManager(exchange_rate_manager.ExchangeRateManager):
|
||||
feed.market, rates[feed.market]['spot'], rates[feed.market]['ts'])
|
||||
|
||||
|
||||
def get_dummy_exchange_rate_manager(time):
|
||||
rates = {
|
||||
'BTCLBC': {'spot': 3.0, 'ts': time.time() + 1},
|
||||
'USDBTC': {'spot': 2.0, 'ts': time.time() + 2}
|
||||
}
|
||||
return DummyExchangeRateManager([BTCLBCFeed(), USDBTCFeed()], rates)
|
||||
|
||||
|
||||
class FeeFormatTest(unittest.TestCase):
|
||||
def test_fee_created_with_correct_inputs(self):
|
||||
fee_dict = {
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
import unittest
|
||||
from lbrynet.extras.daemon import mime_types
|
||||
|
||||
|
||||
class TestMimeTypes(unittest.TestCase):
|
||||
def test_mp4_video(self):
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4"))
|
||||
|
||||
def test_x_ext_(self):
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry"))
|
||||
|
||||
def test_octet_stream(self):
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test."))
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test"))
|
||||
@@ -2,15 +2,13 @@ import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
|
||||
|
||||
class TestStreamAssembler(AsyncioTestCase):
|
||||
@@ -19,10 +17,10 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.key = b'deadbeef' * 4
|
||||
self.cleartext = b'test'
|
||||
|
||||
async def test_create_and_decrypt_one_blob_stream(self, corrupt=False):
|
||||
async def test_create_and_decrypt_one_blob_stream(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), ":memory:")
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
|
||||
@@ -42,11 +40,6 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
for blob_info in sd.blobs:
|
||||
if blob_info.blob_hash:
|
||||
shutil.copy(os.path.join(tmp_dir, blob_info.blob_hash), os.path.join(download_dir, blob_info.blob_hash))
|
||||
if corrupt and blob_info.length == MAX_BLOB_SIZE:
|
||||
with open(os.path.join(download_dir, blob_info.blob_hash), "rb+") as handle:
|
||||
handle.truncate()
|
||||
handle.flush()
|
||||
|
||||
downloader_storage = SQLiteStorage(Config(), os.path.join(download_dir, "lbrynet.sqlite"))
|
||||
await downloader_storage.open()
|
||||
|
||||
@@ -57,19 +50,11 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
# assemble the decrypted file
|
||||
assembler = StreamAssembler(self.loop, downloader_blob_manager, descriptor.sd_hash)
|
||||
await assembler.assemble_decrypted_stream(download_dir)
|
||||
if corrupt:
|
||||
return self.assertFalse(os.path.isfile(os.path.join(download_dir, "test_file")))
|
||||
|
||||
with open(os.path.join(download_dir, "test_file"), "rb") as f:
|
||||
decrypted = f.read()
|
||||
self.assertEqual(decrypted, self.cleartext)
|
||||
self.assertEqual(True, self.blob_manager.get_blob(sd_hash).get_is_verified())
|
||||
self.assertEqual(True, self.blob_manager.get_blob(descriptor.blobs[0].blob_hash).get_is_verified())
|
||||
# its all blobs + sd blob - last blob, which is the same size as descriptor.blobs
|
||||
self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs()))
|
||||
self.assertEqual(
|
||||
[descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce()
|
||||
)
|
||||
|
||||
await downloader_storage.close()
|
||||
await self.storage.close()
|
||||
@@ -90,28 +75,3 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
async def test_create_and_decrypt_random(self):
|
||||
self.cleartext = os.urandom(20000000)
|
||||
await self.test_create_and_decrypt_one_blob_stream()
|
||||
|
||||
async def test_create_managed_stream_announces(self):
|
||||
# setup a blob manager
|
||||
storage = SQLiteStorage(Config(), ":memory:")
|
||||
await storage.open()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
blob_manager = BlobFileManager(self.loop, tmp_dir, storage)
|
||||
stream_manager = StreamManager(self.loop, Config(), blob_manager, None, storage, None)
|
||||
# create the stream
|
||||
download_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(download_dir))
|
||||
file_path = os.path.join(download_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(b'testtest')
|
||||
|
||||
stream = await stream_manager.create_stream(file_path)
|
||||
self.assertEqual(
|
||||
[stream.sd_hash, stream.descriptor.blobs[0].blob_hash],
|
||||
await storage.get_blobs_to_announce())
|
||||
|
||||
async def test_create_truncate_and_handle_stream(self):
|
||||
self.cleartext = b'potato' * 1337 * 5279
|
||||
# The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated
|
||||
await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5)
|
||||
@@ -1,12 +1,7 @@
|
||||
import os
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse
|
||||
from lbrynet.blob_exchange.server import BlobServerProtocol
|
||||
from lbrynet.conf import Config
|
||||
import contextlib
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.dht.node import Node
|
||||
@@ -26,77 +21,61 @@ class TestStreamDownloader(BlobExchangeTestBase):
|
||||
f.write(self.stream_bytes)
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
|
||||
self.sd_hash = descriptor.calculate_sd_hash()
|
||||
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
|
||||
self.downloader = StreamDownloader(self.loop, self.client_blob_manager, self.sd_hash, 3, 3, self.client_dir)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_peer_search=None):
|
||||
await self.setup_stream(blob_count)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
q2.put_nowait([self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
@contextlib.asynccontextmanager
|
||||
async def _mock_peer_search(*_):
|
||||
async def _gen():
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
|
||||
yield _gen()
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search or _mock_peer_search
|
||||
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
self.assertTrue(self.downloader.stream_handle.closed)
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
self.downloader.stop()
|
||||
self.assertIs(self.downloader.stream_handle, None)
|
||||
await self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_transfer_stream(self):
|
||||
await self._test_transfer_stream(10)
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_transfer_hundred_blob_stream(self):
|
||||
await self._test_transfer_stream(100)
|
||||
# async def test_transfer_hundred_blob_stream(self):
|
||||
# await self._test_transfer_stream(100)
|
||||
|
||||
async def test_transfer_stream_bad_first_peer_good_second(self):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
@contextlib.asynccontextmanager
|
||||
async def mock_peer_search(*_):
|
||||
async def _gen():
|
||||
await asyncio.sleep(0.05, loop=self.loop)
|
||||
yield [bad_peer]
|
||||
await asyncio.sleep(0.1, loop=self.loop)
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
|
||||
q2.put_nowait([bad_peer])
|
||||
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
yield _gen()
|
||||
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
mock_node.stream_peer_search_junction = mock_peer_search
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
# self.assertIs(self.server_from_client.tcp_last_down, None)
|
||||
# self.assertIsNot(bad_peer.tcp_last_down, None)
|
||||
|
||||
async def test_client_chunked_response(self):
|
||||
self.server.stop_server()
|
||||
class ChunkedServerProtocol(BlobServerProtocol):
|
||||
|
||||
def send_response(self, responses):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
for byte in BlobResponse(to_send).serialize():
|
||||
self.transport.write(bytes([byte]))
|
||||
self.server.server_protocol_class = ChunkedServerProtocol
|
||||
self.server.start_server(33333, '127.0.0.1')
|
||||
self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes))
|
||||
await asyncio.wait_for(self._test_transfer_stream(10), timeout=2)
|
||||
self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes))
|
||||
@@ -21,7 +21,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
|
||||
self.stream_manager = StreamManager(self.loop, self.blob_manager, None, self.storage, None, 3.0, 3.0)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
import json
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.error import InvalidStreamDescriptorError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
|
||||
class TestStreamDescriptor(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.key = b'deadbeef' * 4
|
||||
self.cleartext = os.urandom(20000000)
|
||||
self.tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), ":memory:")
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, self.tmp_dir, self.storage)
|
||||
|
||||
self.file_path = os.path.join(self.tmp_dir, "test_file")
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(self.cleartext)
|
||||
|
||||
self.descriptor = await StreamDescriptor.create_stream(self.loop, self.tmp_dir, self.file_path, key=self.key)
|
||||
self.sd_hash = self.descriptor.calculate_sd_hash()
|
||||
self.sd_dict = json.loads(self.descriptor.as_json())
|
||||
|
||||
def _write_sd(self):
|
||||
with open(os.path.join(self.tmp_dir, self.sd_hash), 'wb') as f:
|
||||
f.write(json.dumps(self.sd_dict, sort_keys=True).encode())
|
||||
|
||||
async def _test_invalid_sd(self):
|
||||
self._write_sd()
|
||||
with self.assertRaises(InvalidStreamDescriptorError):
|
||||
await self.blob_manager.get_stream_descriptor(self.sd_hash)
|
||||
|
||||
async def test_load_sd_blob(self):
|
||||
self._write_sd()
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(self.sd_hash)
|
||||
self.assertEqual(descriptor.calculate_sd_hash(), self.sd_hash)
|
||||
|
||||
async def test_missing_terminator(self):
|
||||
self.sd_dict['blobs'].pop()
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_terminator_not_at_end(self):
|
||||
terminator = self.sd_dict['blobs'].pop()
|
||||
self.sd_dict['blobs'] = [terminator] + self.sd_dict['blobs']
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_terminator_has_blob_hash(self):
|
||||
self.sd_dict['blobs'][-1]['blob_hash'] = '1' * 96
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_blob_order(self):
|
||||
terminator = self.sd_dict['blobs'].pop()
|
||||
self.sd_dict['blobs'].reverse()
|
||||
self.sd_dict['blobs'].append(terminator)
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_skip_blobs(self):
|
||||
self.sd_dict['blobs'][-2]['blob_num'] = self.sd_dict['blobs'][-2]['blob_num'] + 1
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_invalid_stream_hash(self):
|
||||
self.sd_dict['blobs'][-2]['blob_hash'] = '1' * 96
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_zero_length_blob(self):
|
||||
self.sd_dict['blobs'][-2]['length'] = 0
|
||||
await self._test_invalid_sd()
|
||||
|
||||
|
||||
class TestRecoverOldStreamDescriptors(AsyncioTestCase):
|
||||
async def test_old_key_sort_sd_blob(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
storage = SQLiteStorage(Config(), ":memory:")
|
||||
await storage.open()
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
|
||||
sd_bytes = b'{"stream_name": "4f62616d6120446f6e6b65792d322e73746c", "blobs": [{"length": 1153488, "blob_num' \
|
||||
b'": 0, "blob_hash": "9fa32a249ce3f2d4e46b78599800f368b72f2a7f22b81df443c7f6bdbef496bd61b4c0079c7' \
|
||||
b'3d79c8bb9be9a6bf86592", "iv": "0bf348867244019c9e22196339016ea6"}, {"length": 0, "blob_num": 1,' \
|
||||
b' "iv": "9f36abae16955463919b07ed530a3d18"}], "stream_type": "lbryfile", "key": "a03742b87628aa7' \
|
||||
b'228e48f1dcd207e48", "suggested_file_name": "4f62616d6120446f6e6b65792d322e73746c", "stream_hash' \
|
||||
b'": "b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e1' \
|
||||
b'60207"}'
|
||||
sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2'
|
||||
stream_hash = 'b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e160207'
|
||||
|
||||
blob = blob_manager.get_blob(sd_hash)
|
||||
blob.set_length(len(sd_bytes))
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(sd_bytes)
|
||||
await blob.verified.wait()
|
||||
descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
loop, blob_manager.blob_dir, blob
|
||||
)
|
||||
self.assertEqual(stream_hash, descriptor.get_stream_hash())
|
||||
self.assertEqual(sd_hash, descriptor.calculate_old_sort_sd_hash())
|
||||
self.assertNotEqual(sd_hash, descriptor.calculate_sd_hash())
|
||||
@@ -1,194 +0,0 @@
|
||||
import os
|
||||
import binascii
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
import time
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
from tests.unit.lbrynet_daemon.test_ExchangeRateManager import get_dummy_exchange_rate_manager
|
||||
from lbrynet.error import InsufficientFundsError, KeyFeeAboveMaxAllowed
|
||||
from lbrynet.extras.wallet.manager import LbryWalletManager
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
|
||||
def get_mock_node(peer):
|
||||
def mock_accumulate_peers(q1: asyncio.Queue, q2: asyncio.Queue):
|
||||
async def _task():
|
||||
pass
|
||||
|
||||
q2.put_nowait([peer])
|
||||
return q2, asyncio.create_task(_task())
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
mock_node.accumulate_peers = mock_accumulate_peers
|
||||
mock_node.joined = asyncio.Event()
|
||||
mock_node.joined.set()
|
||||
return mock_node
|
||||
|
||||
|
||||
def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
claim = {
|
||||
"address": "bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF",
|
||||
"amount": "0.1",
|
||||
"claim_id": "c49566d631226492317d06ad7fdbe1ed32925124",
|
||||
"claim_sequence": 1,
|
||||
"decoded_claim": True,
|
||||
"depth": 1057,
|
||||
"effective_amount": "0.1",
|
||||
"has_signature": False,
|
||||
"height": 514081,
|
||||
"hex": "",
|
||||
"name": "33rpm",
|
||||
"nout": 0,
|
||||
"permanent_url": "33rpm#c49566d631226492317d06ad7fdbe1ed32925124",
|
||||
"supports": [],
|
||||
"txid": "81ac52662af926fdf639d56920069e0f63449d4cde074c61717cb99ddde40e3c",
|
||||
"value": {
|
||||
"claimType": "streamType",
|
||||
"stream": {
|
||||
"metadata": {
|
||||
"author": "",
|
||||
"description": "",
|
||||
"language": "en",
|
||||
"license": "None",
|
||||
"licenseUrl": "",
|
||||
"nsfw": False,
|
||||
"preview": "",
|
||||
"thumbnail": "",
|
||||
"title": "33rpm",
|
||||
"version": "_0_1_0"
|
||||
},
|
||||
"source": {
|
||||
"contentType": "image/png",
|
||||
"source": sd_hash,
|
||||
"sourceType": "lbry_sd_hash",
|
||||
"version": "_0_0_1"
|
||||
},
|
||||
"version": "_0_0_1"
|
||||
},
|
||||
"version": "_0_0_1"
|
||||
}
|
||||
}
|
||||
if fee:
|
||||
claim['value']['stream']['metadata']['fee'] = fee
|
||||
claim_dict = ClaimDict.load_dict(claim['value'])
|
||||
claim['hex'] = binascii.hexlify(claim_dict.serialized).decode()
|
||||
|
||||
async def mock_resolve(*args):
|
||||
await storage.save_claims([claim])
|
||||
return {
|
||||
claim['permanent_url']: claim
|
||||
}
|
||||
|
||||
mock_wallet = mock.Mock(spec=LbryWalletManager)
|
||||
mock_wallet.resolve = mock_resolve
|
||||
|
||||
async def get_balance(*_):
|
||||
return balance
|
||||
|
||||
mock_wallet.default_account.get_balance = get_balance
|
||||
return mock_wallet, claim['permanent_url']
|
||||
|
||||
|
||||
class TestStreamManager(BlobExchangeTestBase):
|
||||
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
|
||||
file_path = os.path.join(self.server_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(os.urandom(20000000))
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path,
|
||||
old_sort=old_sort)
|
||||
self.sd_hash = descriptor.sd_hash
|
||||
self.mock_wallet, self.uri = get_mock_wallet(self.sd_hash, self.client_storage, balance, fee)
|
||||
self.stream_manager = StreamManager(self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
|
||||
self.client_storage, get_mock_node(self.server_from_client))
|
||||
self.exchange_rate_manager = get_dummy_exchange_rate_manager(time)
|
||||
|
||||
async def test_download_stop_resume_delete(self):
|
||||
await self.setup_stream_manager()
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream_hash = stream.stream_hash
|
||||
self.assertSetEqual(self.stream_manager.streams, {stream})
|
||||
self.assertTrue(stream.running)
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, "running")
|
||||
|
||||
await self.stream_manager.stop_stream(stream)
|
||||
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, "stopped")
|
||||
|
||||
await self.stream_manager.start_stream(stream)
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
await asyncio.sleep(0.01)
|
||||
self.assertTrue(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, "finished")
|
||||
|
||||
await self.stream_manager.delete_stream(stream, True)
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, None)
|
||||
|
||||
async def test_insufficient_funds(self):
|
||||
fee = {
|
||||
'currency': 'LBC',
|
||||
'amount': 11.0,
|
||||
'address': 'bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF',
|
||||
'version': '_0_0_1'
|
||||
}
|
||||
await self.setup_stream_manager(10.0, fee)
|
||||
with self.assertRaises(InsufficientFundsError):
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
|
||||
async def test_fee_above_max_allowed(self):
|
||||
fee = {
|
||||
'currency': 'USD',
|
||||
'amount': 51.0,
|
||||
'address': 'bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF',
|
||||
'version': '_0_0_1'
|
||||
}
|
||||
await self.setup_stream_manager(1000000.0, fee)
|
||||
with self.assertRaises(KeyFeeAboveMaxAllowed):
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
|
||||
async def test_download_then_recover_stream_on_startup(self, old_sort=False):
|
||||
await self.setup_stream_manager(old_sort=old_sort)
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
self.stream_manager.stop()
|
||||
self.client_blob_manager.stop()
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, stream.sd_hash))
|
||||
for blob in stream.descriptor.blobs[:-1]:
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, blob.blob_hash))
|
||||
await self.client_blob_manager.setup()
|
||||
await self.stream_manager.start()
|
||||
self.assertEqual(1, len(self.stream_manager.streams))
|
||||
self.assertEqual(stream.sd_hash, list(self.stream_manager.streams)[0].sd_hash)
|
||||
self.assertEqual('stopped', list(self.stream_manager.streams)[0].status)
|
||||
|
||||
sd_blob = self.client_blob_manager.get_blob(stream.sd_hash)
|
||||
self.assertTrue(sd_blob.file_exists)
|
||||
self.assertTrue(sd_blob.get_is_verified())
|
||||
|
||||
def test_download_then_recover_old_sort_stream_on_startup(self):
|
||||
return self.test_download_then_recover_stream_on_startup(old_sort=True)
|
||||
@@ -12,4 +12,5 @@ setenv =
|
||||
TORBA_LEDGER=lbrynet.extras.wallet
|
||||
commands =
|
||||
orchstr8 download
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration.wallet
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration.cli
|
||||
Reference in new issue
Block a user