Compare commits
87
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91e581422c | ||
|
|
b54d7185a1 | ||
|
|
f2968aab22 | ||
|
|
4e27eaf8e6 | ||
|
|
ccbe8aa0eb | ||
|
|
478f79353a | ||
|
|
c3b0151943 | ||
|
|
ba818354af | ||
|
|
e96b75a0d0 | ||
|
|
efe4afd09e | ||
|
|
c7cb6822cc | ||
|
|
c75665d3f0 | ||
|
|
744375b2c0 | ||
|
|
c1b4a012ec | ||
|
|
40da5b6cb6 | ||
|
|
b36cf3e36c | ||
|
|
4e3b529aec | ||
|
|
0a08f445a4 | ||
|
|
3eac99c509 | ||
|
|
f5bf1463af | ||
|
|
ca4a55ef28 | ||
|
|
832be0758b | ||
|
|
25050fdeeb | ||
|
|
df0635103e | ||
|
|
dabd9cbe1e | ||
|
|
42d7fbdbb3 | ||
|
|
2336015f9a | ||
|
|
5a28128b69 | ||
|
|
31445c7797 | ||
|
|
16d0ff8376 | ||
|
|
87f5d1bd4a | ||
|
|
be6e8744b0 | ||
|
|
2e9acd1baa | ||
|
|
607ccefe90 | ||
|
|
df86937f4e | ||
|
|
957d0290df | ||
|
|
40b90ce79b | ||
|
|
aa7f0845cc | ||
|
|
6e69d306de | ||
|
|
d85d3dfaa4 | ||
|
|
2b035009ef | ||
|
|
f9fd62c214 | ||
|
|
e8a1953815 | ||
|
|
34776e7eb9 | ||
|
|
a4419f562f | ||
|
|
c0160ab4e9 | ||
|
|
e60e5b3919 | ||
|
|
3589cc9977 | ||
|
|
2e978c00b2 | ||
|
|
2335243de9 | ||
|
|
6ad68eabd9 | ||
|
|
ca5c638124 | ||
|
|
262b9a624b | ||
|
|
05915c540a | ||
|
|
6aef6a80b7 | ||
|
|
16efe9ba95 | ||
|
|
fcda9500fe | ||
|
|
64dec821b7 | ||
|
|
d5f6e58aad | ||
|
|
087b65522e | ||
|
|
5ee4b2173a | ||
|
|
3e1c3d1e4b | ||
|
|
21f2b2dcd7 | ||
|
|
ffc7fc0793 | ||
|
|
15dbe5e54e | ||
|
|
2e633c0ee6 | ||
|
|
933d58d49b | ||
|
|
8f7cf0b38f | ||
|
|
adf378b6e9 | ||
|
|
1a4f083e76 | ||
|
|
7bde09dcf8 | ||
|
|
4630fd38aa | ||
|
|
418def56ac | ||
|
|
cefe3eb520 | ||
|
|
64dffa306f | ||
|
|
7d33b4f1f3 | ||
|
|
f507d95198 | ||
|
|
2a7d80f8b4 | ||
|
|
305034ffb2 | ||
|
|
bfaebf0085 | ||
|
|
a94cf04e87 | ||
|
|
71f9f8ae9c | ||
|
|
330862e487 | ||
|
|
778d3826ab | ||
|
|
4a11cf007f | ||
|
|
f6e7700c4c | ||
|
|
573285a5bb |
No files matched your search
@@ -1,12 +1,12 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
build:
|
||||
python setup.py apk
|
||||
adb install -r lbry-0.1-service-debug.apk
|
||||
|
||||
.PHONY: build
|
||||
@@ -1,49 +0,0 @@
|
||||
import inspect
|
||||
from os.path import dirname, join
|
||||
|
||||
import sh
|
||||
from pythonforandroid.util import ensure_dir
|
||||
from pythonforandroid.toolchain import (
|
||||
Bootstrap, current_directory, info, info_main, shprint
|
||||
)
|
||||
|
||||
|
||||
class LBRYServiceBootstrap(Bootstrap):
|
||||
name = 'lbry-service'
|
||||
recipe_depends = ['genericndkbuild', 'python3']
|
||||
bootstrap_dir = dirname(__file__)
|
||||
|
||||
def get_common_dir(self):
|
||||
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
|
||||
|
||||
def run_distribute(self):
|
||||
info_main('# Creating Android project from build and {} bootstrap'.format(
|
||||
self.name))
|
||||
|
||||
info('This currently just copies the build stuff straight from the build dir.')
|
||||
shprint(sh.rm, '-rf', self.dist_dir)
|
||||
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
|
||||
with current_directory(self.dist_dir):
|
||||
with open('local.properties', 'w') as fileh:
|
||||
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
|
||||
|
||||
arch = self.ctx.archs[0]
|
||||
if len(self.ctx.archs) > 1:
|
||||
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
|
||||
info('Bootstrap running with arch {}'.format(arch))
|
||||
|
||||
with current_directory(self.dist_dir):
|
||||
info('Copying python distribution')
|
||||
|
||||
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
|
||||
self.distribute_aars(arch)
|
||||
self.distribute_javaclasses(self.ctx.javaclass_dir)
|
||||
|
||||
python_bundle_dir = join('_python_bundle', '_python_bundle')
|
||||
ensure_dir(python_bundle_dir)
|
||||
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
|
||||
join(self.dist_dir, python_bundle_dir), arch)
|
||||
|
||||
self.strip_libraries(arch)
|
||||
self.fry_eggs(site_packages_dir)
|
||||
super().run_distribute()
|
||||
@@ -1,89 +0,0 @@
|
||||
# prevent user to include invalid extensions
|
||||
*.apk
|
||||
*.pxd
|
||||
|
||||
# eggs
|
||||
*.egg-info
|
||||
|
||||
# unit test
|
||||
unittest/*
|
||||
|
||||
# python config
|
||||
config/makesetup
|
||||
|
||||
# unused kivy files (platform specific)
|
||||
kivy/input/providers/wm_*
|
||||
kivy/input/providers/mactouch*
|
||||
kivy/input/providers/probesysfs*
|
||||
kivy/input/providers/mtdev*
|
||||
kivy/input/providers/hidinput*
|
||||
kivy/core/camera/camera_videocapture*
|
||||
kivy/core/spelling/*osx*
|
||||
kivy/core/video/video_pyglet*
|
||||
kivy/tools
|
||||
kivy/tests/*
|
||||
kivy/*/*.h
|
||||
kivy/*/*.pxi
|
||||
|
||||
# unused encodings
|
||||
lib-dynload/*codec*
|
||||
encodings/cp*.pyo
|
||||
encodings/tis*
|
||||
encodings/shift*
|
||||
encodings/bz2*
|
||||
encodings/iso*
|
||||
encodings/undefined*
|
||||
encodings/johab*
|
||||
encodings/p*
|
||||
encodings/m*
|
||||
encodings/euc*
|
||||
encodings/k*
|
||||
encodings/unicode_internal*
|
||||
encodings/quo*
|
||||
encodings/gb*
|
||||
encodings/big5*
|
||||
encodings/hp*
|
||||
encodings/hz*
|
||||
|
||||
# unused python modules
|
||||
bsddb/*
|
||||
wsgiref/*
|
||||
hotshot/*
|
||||
pydoc_data/*
|
||||
tty.pyo
|
||||
anydbm.pyo
|
||||
nturl2path.pyo
|
||||
LICENCE.txt
|
||||
macurl2path.pyo
|
||||
dummy_threading.pyo
|
||||
audiodev.pyo
|
||||
antigravity.pyo
|
||||
dumbdbm.pyo
|
||||
sndhdr.pyo
|
||||
__phello__.foo.pyo
|
||||
sunaudio.pyo
|
||||
os2emxpath.pyo
|
||||
multiprocessing/dummy*
|
||||
|
||||
# unused binaries python modules
|
||||
lib-dynload/termios.so
|
||||
lib-dynload/_lsprof.so
|
||||
lib-dynload/*audioop.so
|
||||
lib-dynload/_hotshot.so
|
||||
lib-dynload/_heapq.so
|
||||
lib-dynload/_json.so
|
||||
lib-dynload/grp.so
|
||||
lib-dynload/resource.so
|
||||
lib-dynload/pyexpat.so
|
||||
lib-dynload/_ctypes_test.so
|
||||
lib-dynload/_testcapi.so
|
||||
|
||||
# odd files
|
||||
plat-linux3/regen
|
||||
|
||||
#>sqlite3
|
||||
# conditionnal include depending if some recipes are included or not.
|
||||
sqlite3/*
|
||||
lib-dynload/_sqlite3.so
|
||||
#<sqlite3
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
include $(call all-subdir-makefiles)
|
||||
@@ -1,7 +0,0 @@
|
||||
|
||||
# Uncomment this if you're using STL in your project
|
||||
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
|
||||
# APP_STL := stlport_static
|
||||
|
||||
# APP_ABI := armeabi armeabi-v7a x86
|
||||
APP_ABI := $(ARCH)
|
||||
@@ -1,22 +0,0 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
# Add your application source files here...
|
||||
LOCAL_SRC_FILES := start.c pyjniusjni.c
|
||||
|
||||
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
|
||||
|
||||
LOCAL_SHARED_LIBRARIES := python_shared
|
||||
|
||||
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
|
||||
|
||||
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
|
||||
ifdef CRYSTAX_PYTHON_VERSION
|
||||
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
|
||||
endif
|
||||
@@ -1,10 +0,0 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
LOCAL_SRC_FILES := YourSourceHere.c
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
|
||||
@@ -1,6 +0,0 @@
|
||||
|
||||
#define BOOTSTRAP_NAME_SERVICEONLY
|
||||
#define BOOTSTRAP_USES_NO_SDL_HEADERS
|
||||
|
||||
const char bootstrap_name[] = "service_only";
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
|
||||
#include <pthread.h>
|
||||
#include <jni.h>
|
||||
|
||||
#define LOGI(...) do {} while (0)
|
||||
#define LOGE(...) do {} while (0)
|
||||
|
||||
#include "android/log.h"
|
||||
|
||||
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
|
||||
|
||||
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
|
||||
/* #define LOGP(x) LOG("python", (x)) */
|
||||
#define LOG_TAG "Python_android"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
|
||||
|
||||
|
||||
/* Function headers */
|
||||
JNIEnv* Android_JNI_GetEnv(void);
|
||||
static void Android_JNI_ThreadDestroyed(void*);
|
||||
|
||||
static pthread_key_t mThreadKey;
|
||||
static JavaVM* mJavaVM;
|
||||
|
||||
int Android_JNI_SetupThread(void)
|
||||
{
|
||||
Android_JNI_GetEnv();
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Library init */
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
|
||||
{
|
||||
JNIEnv *env;
|
||||
mJavaVM = vm;
|
||||
LOGI("JNI_OnLoad called");
|
||||
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
|
||||
LOGE("Failed to get the environment using GetEnv()");
|
||||
return -1;
|
||||
}
|
||||
/*
|
||||
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
|
||||
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
|
||||
*/
|
||||
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
|
||||
|
||||
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
|
||||
}
|
||||
Android_JNI_SetupThread();
|
||||
|
||||
return JNI_VERSION_1_4;
|
||||
}
|
||||
|
||||
JNIEnv* Android_JNI_GetEnv(void)
|
||||
{
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* All threads are Linux threads, scheduled by the kernel.
|
||||
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
|
||||
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
|
||||
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
|
||||
* and cannot make JNI calls.
|
||||
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
|
||||
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
|
||||
* is a no-op.
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
*/
|
||||
|
||||
JNIEnv *env;
|
||||
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
|
||||
if(status < 0) {
|
||||
LOGE("failed to attach current thread");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
|
||||
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
|
||||
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
|
||||
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
|
||||
* Note: The destructor is not called unless the stored value is != NULL
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
* (except for some lost CPU cycles)
|
||||
*/
|
||||
pthread_setspecific(mThreadKey, (void*) env);
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
static void Android_JNI_ThreadDestroyed(void* value)
|
||||
{
|
||||
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
|
||||
JNIEnv *env = (JNIEnv*) value;
|
||||
if (env != NULL) {
|
||||
(*mJavaVM)->DetachCurrentThread(mJavaVM);
|
||||
pthread_setspecific(mThreadKey, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
void *WebView_AndroidGetJNIEnv()
|
||||
{
|
||||
return Android_JNI_GetEnv();
|
||||
}
|
||||
Whitespace-only changes.
@@ -1,141 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class Octal {
|
||||
|
||||
/**
|
||||
* Parse an octal string from a header buffer. This is used for the file
|
||||
* permission mode value.
|
||||
*
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal string.
|
||||
*/
|
||||
public static long parseOctal(byte[] header, int offset, int length) {
|
||||
long result = 0;
|
||||
boolean stillPadding = true;
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
|
||||
if (header[i] == (byte) ' ' || header[i] == '0') {
|
||||
if (stillPadding)
|
||||
continue;
|
||||
|
||||
if (header[i] == (byte) ' ')
|
||||
break;
|
||||
}
|
||||
|
||||
stillPadding = false;
|
||||
|
||||
result = ( result << 3 ) + ( header[i] - '0' );
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The integer value of the octal bytes.
|
||||
*/
|
||||
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
int idx = length - 1;
|
||||
|
||||
buf[offset + idx] = 0;
|
||||
--idx;
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
--idx;
|
||||
|
||||
if (value == 0) {
|
||||
buf[offset + idx] = (byte) '0';
|
||||
--idx;
|
||||
} else {
|
||||
for (long val = value; idx >= 0 && val > 0; --idx) {
|
||||
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
|
||||
val = val >> 3;
|
||||
}
|
||||
}
|
||||
|
||||
for (; idx >= 0; --idx) {
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the checksum octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The integer value of the entry's checksum.
|
||||
*/
|
||||
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
getOctalBytes( value, buf, offset, length );
|
||||
buf[offset + length - 1] = (byte) ' ';
|
||||
buf[offset + length - 2] = 0;
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal long integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal bytes.
|
||||
*/
|
||||
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
byte[] temp = new byte[length + 1];
|
||||
getOctalBytes( value, temp, 0, length + 1 );
|
||||
System.arraycopy( temp, 0, buf, offset, length );
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarConstants {
|
||||
public static final int EOF_BLOCK = 1024;
|
||||
public static final int DATA_BLOCK = 512;
|
||||
public static final int HEADER_BLOCK = 512;
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarEntry {
|
||||
protected File file;
|
||||
protected TarHeader header;
|
||||
|
||||
private TarEntry() {
|
||||
this.file = null;
|
||||
header = new TarHeader();
|
||||
}
|
||||
|
||||
public TarEntry(File file, String entryName) {
|
||||
this();
|
||||
this.file = file;
|
||||
this.extractTarHeader(entryName);
|
||||
}
|
||||
|
||||
public TarEntry(byte[] headerBuf) {
|
||||
this();
|
||||
this.parseTarHeader(headerBuf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor to create an entry from an existing TarHeader object.
|
||||
*
|
||||
* This method is useful to add new entries programmatically (e.g. for
|
||||
* adding files or directories that do not exist in the file system).
|
||||
*
|
||||
* @param header
|
||||
*
|
||||
*/
|
||||
public TarEntry(TarHeader header) {
|
||||
this.file = null;
|
||||
this.header = header;
|
||||
}
|
||||
|
||||
public boolean equals(TarEntry it) {
|
||||
return header.name.toString().equals(it.header.name.toString());
|
||||
}
|
||||
|
||||
public boolean isDescendent(TarEntry desc) {
|
||||
return desc.header.name.toString().startsWith(header.name.toString());
|
||||
}
|
||||
|
||||
public TarHeader getHeader() {
|
||||
return header;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
String name = header.name.toString();
|
||||
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
|
||||
name = header.namePrefix.toString() + "/" + name;
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
public int getUserId() {
|
||||
return header.userId;
|
||||
}
|
||||
|
||||
public void setUserId(int userId) {
|
||||
header.userId = userId;
|
||||
}
|
||||
|
||||
public int getGroupId() {
|
||||
return header.groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(int groupId) {
|
||||
header.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return header.userName.toString();
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
header.userName = new StringBuffer(userName);
|
||||
}
|
||||
|
||||
public String getGroupName() {
|
||||
return header.groupName.toString();
|
||||
}
|
||||
|
||||
public void setGroupName(String groupName) {
|
||||
header.groupName = new StringBuffer(groupName);
|
||||
}
|
||||
|
||||
public void setIds(int userId, int groupId) {
|
||||
this.setUserId(userId);
|
||||
this.setGroupId(groupId);
|
||||
}
|
||||
|
||||
public void setModTime(long time) {
|
||||
header.modTime = time / 1000;
|
||||
}
|
||||
|
||||
public void setModTime(Date time) {
|
||||
header.modTime = time.getTime() / 1000;
|
||||
}
|
||||
|
||||
public Date getModTime() {
|
||||
return new Date(header.modTime * 1000);
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return this.file;
|
||||
}
|
||||
|
||||
public long getSize() {
|
||||
return header.size;
|
||||
}
|
||||
|
||||
public void setSize(long size) {
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the org.kamrazafar.jtar entry is a directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean isDirectory() {
|
||||
if (this.file != null)
|
||||
return this.file.isDirectory();
|
||||
|
||||
if (header != null) {
|
||||
if (header.linkFlag == TarHeader.LF_DIR)
|
||||
return true;
|
||||
|
||||
if (header.name.toString().endsWith("/"))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract header from File
|
||||
*
|
||||
* @param entryName
|
||||
*/
|
||||
public void extractTarHeader(String entryName) {
|
||||
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum
|
||||
*
|
||||
* @param buf
|
||||
* @return
|
||||
*/
|
||||
public long computeCheckSum(byte[] buf) {
|
||||
long sum = 0;
|
||||
|
||||
for (int i = 0; i < buf.length; ++i) {
|
||||
sum += 255 & buf[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the header to the byte buffer
|
||||
*
|
||||
* @param outbuf
|
||||
*/
|
||||
public void writeEntryHeader(byte[] outbuf) {
|
||||
int offset = 0;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
|
||||
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
|
||||
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
|
||||
|
||||
long size = header.size;
|
||||
|
||||
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
|
||||
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
|
||||
|
||||
int csOffset = offset;
|
||||
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
|
||||
outbuf[offset++] = (byte) ' ';
|
||||
|
||||
outbuf[offset++] = header.linkFlag;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
|
||||
for (; offset < outbuf.length;)
|
||||
outbuf[offset++] = 0;
|
||||
|
||||
long checkSum = this.computeCheckSum(outbuf);
|
||||
|
||||
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the tar header to the byte buffer
|
||||
*
|
||||
* @param header
|
||||
* @param bh
|
||||
*/
|
||||
public void parseTarHeader(byte[] bh) {
|
||||
int offset = 0;
|
||||
|
||||
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
|
||||
offset += TarHeader.MODELEN;
|
||||
|
||||
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
|
||||
offset += TarHeader.UIDLEN;
|
||||
|
||||
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
|
||||
offset += TarHeader.GIDLEN;
|
||||
|
||||
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
|
||||
offset += TarHeader.SIZELEN;
|
||||
|
||||
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
|
||||
offset += TarHeader.MODTIMELEN;
|
||||
|
||||
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
|
||||
offset += TarHeader.CHKSUMLEN;
|
||||
|
||||
header.linkFlag = bh[offset++];
|
||||
|
||||
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset += TarHeader.USTAR_MAGICLEN;
|
||||
|
||||
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset += TarHeader.USTAR_USER_NAMELEN;
|
||||
|
||||
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset += TarHeader.USTAR_GROUP_NAMELEN;
|
||||
|
||||
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
}
|
||||
}
|
||||
@@ -1,243 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 0 100 File name
|
||||
* 100 8 File mode
|
||||
* 108 8 Owner's numeric user ID
|
||||
* 116 8 Group's numeric user ID
|
||||
* 124 12 File size in bytes
|
||||
* 136 12 Last modification time in numeric Unix time format
|
||||
* 148 8 Checksum for header block
|
||||
* 156 1 Link indicator (file type)
|
||||
* 157 100 Name of linked file
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* File Types
|
||||
*
|
||||
* <pre>
|
||||
* Value Meaning
|
||||
* '0' Normal file
|
||||
* (ASCII NUL) Normal file (now obsolete)
|
||||
* '1' Hard link
|
||||
* '2' Symbolic link
|
||||
* '3' Character special
|
||||
* '4' Block special
|
||||
* '5' Directory
|
||||
* '6' FIFO
|
||||
* '7' Contigous
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
*
|
||||
* Ustar header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 257 6 UStar indicator "ustar"
|
||||
* 263 2 UStar version "00"
|
||||
* 265 32 Owner user name
|
||||
* 297 32 Owner group name
|
||||
* 329 8 Device major number
|
||||
* 337 8 Device minor number
|
||||
* 345 155 Filename prefix
|
||||
* </pre>
|
||||
*/
|
||||
|
||||
public class TarHeader {
|
||||
|
||||
/*
|
||||
* Header
|
||||
*/
|
||||
public static final int NAMELEN = 100;
|
||||
public static final int MODELEN = 8;
|
||||
public static final int UIDLEN = 8;
|
||||
public static final int GIDLEN = 8;
|
||||
public static final int SIZELEN = 12;
|
||||
public static final int MODTIMELEN = 12;
|
||||
public static final int CHKSUMLEN = 8;
|
||||
public static final byte LF_OLDNORM = 0;
|
||||
|
||||
/*
|
||||
* File Types
|
||||
*/
|
||||
public static final byte LF_NORMAL = (byte) '0';
|
||||
public static final byte LF_LINK = (byte) '1';
|
||||
public static final byte LF_SYMLINK = (byte) '2';
|
||||
public static final byte LF_CHR = (byte) '3';
|
||||
public static final byte LF_BLK = (byte) '4';
|
||||
public static final byte LF_DIR = (byte) '5';
|
||||
public static final byte LF_FIFO = (byte) '6';
|
||||
public static final byte LF_CONTIG = (byte) '7';
|
||||
|
||||
/*
|
||||
* Ustar header
|
||||
*/
|
||||
|
||||
public static final String USTAR_MAGIC = "ustar"; // POSIX
|
||||
|
||||
public static final int USTAR_MAGICLEN = 8;
|
||||
public static final int USTAR_USER_NAMELEN = 32;
|
||||
public static final int USTAR_GROUP_NAMELEN = 32;
|
||||
public static final int USTAR_DEVLEN = 8;
|
||||
public static final int USTAR_FILENAME_PREFIX = 155;
|
||||
|
||||
// Header values
|
||||
public StringBuffer name;
|
||||
public int mode;
|
||||
public int userId;
|
||||
public int groupId;
|
||||
public long size;
|
||||
public long modTime;
|
||||
public int checkSum;
|
||||
public byte linkFlag;
|
||||
public StringBuffer linkName;
|
||||
public StringBuffer magic; // ustar indicator and version
|
||||
public StringBuffer userName;
|
||||
public StringBuffer groupName;
|
||||
public int devMajor;
|
||||
public int devMinor;
|
||||
public StringBuffer namePrefix;
|
||||
|
||||
public TarHeader() {
|
||||
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
|
||||
|
||||
this.name = new StringBuffer();
|
||||
this.linkName = new StringBuffer();
|
||||
|
||||
String user = System.getProperty("user.name", "");
|
||||
|
||||
if (user.length() > 31)
|
||||
user = user.substring(0, 31);
|
||||
|
||||
this.userId = 0;
|
||||
this.groupId = 0;
|
||||
this.userName = new StringBuffer(user);
|
||||
this.groupName = new StringBuffer("");
|
||||
this.namePrefix = new StringBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an entry name from a header buffer.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The header's entry name.
|
||||
*/
|
||||
public static StringBuffer parseName(byte[] header, int offset, int length) {
|
||||
StringBuffer result = new StringBuffer(length);
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
result.append((char) header[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the number of bytes in an entry name.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The number of bytes in a header's entry name.
|
||||
*/
|
||||
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < length && i < name.length(); ++i) {
|
||||
buf[offset + i] = (byte) name.charAt(i);
|
||||
}
|
||||
|
||||
for (; i < length; ++i) {
|
||||
buf[offset + i] = 0;
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new header for a file/directory entry.
|
||||
*
|
||||
*
|
||||
* @param name
|
||||
* File name
|
||||
* @param size
|
||||
* File size in bytes
|
||||
* @param modTime
|
||||
* Last modification time in numeric Unix time format
|
||||
* @param dir
|
||||
* Is directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
|
||||
String name = entryName;
|
||||
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
|
||||
|
||||
TarHeader header = new TarHeader();
|
||||
header.linkName = new StringBuffer("");
|
||||
|
||||
if (name.length() > 100) {
|
||||
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
|
||||
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
|
||||
} else {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
if (dir) {
|
||||
header.mode = 040755;
|
||||
header.linkFlag = TarHeader.LF_DIR;
|
||||
if (header.name.charAt(header.name.length() - 1) != '/') {
|
||||
header.name.append("/");
|
||||
}
|
||||
header.size = 0;
|
||||
} else {
|
||||
header.mode = 0100644;
|
||||
header.linkFlag = TarHeader.LF_NORMAL;
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
header.modTime = modTime;
|
||||
header.checkSum = 0;
|
||||
header.devMajor = 0;
|
||||
header.devMinor = 0;
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
@@ -1,249 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarInputStream extends FilterInputStream {
|
||||
|
||||
private static final int SKIP_BUFFER_SIZE = 2048;
|
||||
private TarEntry currentEntry;
|
||||
private long currentFileSize;
|
||||
private long bytesRead;
|
||||
private boolean defaultSkip = false;
|
||||
|
||||
public TarInputStream(InputStream in) {
|
||||
super(in);
|
||||
currentFileSize = 0;
|
||||
bytesRead = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void mark(int readlimit) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void reset() throws IOException {
|
||||
throw new IOException("mark/reset not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a byte
|
||||
*
|
||||
* @see java.io.FilterInputStream#read()
|
||||
*/
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
byte[] buf = new byte[1];
|
||||
|
||||
int res = this.read(buf, 0, 1);
|
||||
|
||||
if (res != -1) {
|
||||
return 0xFF & buf[0];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being read exceed the entry size and adjusts the byte
|
||||
* array length. Updates the byte counters
|
||||
*
|
||||
*
|
||||
* @see java.io.FilterInputStream#read(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentFileSize == currentEntry.getSize()) {
|
||||
return -1;
|
||||
} else if ((currentEntry.getSize() - currentFileSize) < len) {
|
||||
len = (int) (currentEntry.getSize() - currentFileSize);
|
||||
}
|
||||
}
|
||||
|
||||
int br = super.read(b, off, len);
|
||||
|
||||
if (br != -1) {
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += br;
|
||||
}
|
||||
|
||||
bytesRead += br;
|
||||
}
|
||||
|
||||
return br;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next entry in the tar file
|
||||
*
|
||||
* @return TarEntry
|
||||
* @throws IOException
|
||||
*/
|
||||
public TarEntry getNextEntry() throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
|
||||
int tr = 0;
|
||||
|
||||
// Read full header
|
||||
while (tr < TarConstants.HEADER_BLOCK) {
|
||||
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
|
||||
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
System.arraycopy(theader, 0, header, tr, res);
|
||||
tr += res;
|
||||
}
|
||||
|
||||
// Check if record is null
|
||||
boolean eof = true;
|
||||
for (byte b : header) {
|
||||
if (b != 0) {
|
||||
eof = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!eof) {
|
||||
currentEntry = new TarEntry(header);
|
||||
}
|
||||
|
||||
return currentEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current offset (in bytes) from the beginning of the stream.
|
||||
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
|
||||
*/
|
||||
public long getCurrentOffset() {
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
// Not fully read, skip rest of the bytes
|
||||
long bs = 0;
|
||||
while (bs < currentEntry.getSize() - currentFileSize) {
|
||||
long res = skip(currentEntry.getSize() - currentFileSize - bs);
|
||||
|
||||
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
|
||||
// I suspect file corruption
|
||||
throw new IOException("Possible tar file corruption");
|
||||
}
|
||||
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0L;
|
||||
skipPad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips the pad at the end of each tar entry file content
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void skipPad() throws IOException {
|
||||
if (bytesRead > 0) {
|
||||
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
|
||||
|
||||
if (extra > 0) {
|
||||
long bs = 0;
|
||||
while (bs < TarConstants.DATA_BLOCK - extra) {
|
||||
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips 'n' bytes on the InputStream<br>
|
||||
* Overrides default implementation of skip
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
if (defaultSkip) {
|
||||
// use skip method of parent stream
|
||||
// may not work if skip not implemented by parent
|
||||
long bs = super.skip(n);
|
||||
bytesRead += bs;
|
||||
|
||||
return bs;
|
||||
}
|
||||
|
||||
if (n <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
long left = n;
|
||||
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
|
||||
|
||||
while (left > 0) {
|
||||
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
left -= res;
|
||||
}
|
||||
|
||||
return n - left;
|
||||
}
|
||||
|
||||
public boolean isDefaultSkip() {
|
||||
return defaultSkip;
|
||||
}
|
||||
|
||||
public void setDefaultSkip(boolean defaultSkip) {
|
||||
this.defaultSkip = defaultSkip;
|
||||
}
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarOutputStream extends OutputStream {
|
||||
private final OutputStream out;
|
||||
private long bytesWritten;
|
||||
private long currentFileSize;
|
||||
private TarEntry currentEntry;
|
||||
|
||||
public TarOutputStream(OutputStream out) {
|
||||
this.out = out;
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
public TarOutputStream(final File fout) throws FileNotFoundException {
|
||||
this.out = new BufferedOutputStream(new FileOutputStream(fout));
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a file for writing.
|
||||
*/
|
||||
public TarOutputStream(final File fout, final boolean append) throws IOException {
|
||||
@SuppressWarnings("resource")
|
||||
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
|
||||
final long fileSize = fout.length();
|
||||
if (append && fileSize > TarConstants.EOF_BLOCK) {
|
||||
raf.seek(fileSize - TarConstants.EOF_BLOCK);
|
||||
}
|
||||
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the EOF record and closes the stream
|
||||
*
|
||||
* @see java.io.FilterOutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closeCurrentEntry();
|
||||
write( new byte[TarConstants.EOF_BLOCK] );
|
||||
out.close();
|
||||
}
|
||||
/**
|
||||
* Writes a byte to the stream and updates byte counters
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
out.write( b );
|
||||
bytesWritten += 1;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being written exceed the current entry size.
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null && !currentEntry.isDirectory()) {
|
||||
if (currentEntry.getSize() < currentFileSize + len) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
|
||||
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
|
||||
+ "] being written." );
|
||||
}
|
||||
}
|
||||
|
||||
out.write( b, off, len );
|
||||
|
||||
bytesWritten += len;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += len;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the next tar entry header on the stream
|
||||
*
|
||||
* @param entry
|
||||
* @throws IOException
|
||||
*/
|
||||
public void putNextEntry(TarEntry entry) throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
entry.writeEntryHeader( header );
|
||||
|
||||
write( header );
|
||||
|
||||
currentEntry = entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
|
||||
+ currentEntry.getSize() + "] has not been fully written." );
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0;
|
||||
|
||||
pad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pads the last content block
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void pad() throws IOException {
|
||||
if (bytesWritten > 0) {
|
||||
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
|
||||
|
||||
if (extra > 0) {
|
||||
write( new byte[TarConstants.DATA_BLOCK - extra] );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author Kamran
|
||||
*
|
||||
*/
|
||||
public class TarUtils {
|
||||
/**
|
||||
* Determines the tar file size of the given folder/file path
|
||||
*
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
public static long calculateTarSize(File path) {
|
||||
return tarSize(path) + TarConstants.EOF_BLOCK;
|
||||
}
|
||||
|
||||
private static long tarSize(File dir) {
|
||||
long size = 0;
|
||||
|
||||
if (dir.isFile()) {
|
||||
return entrySize(dir.length());
|
||||
} else {
|
||||
File[] subFiles = dir.listFiles();
|
||||
|
||||
if (subFiles != null && subFiles.length > 0) {
|
||||
for (File file : subFiles) {
|
||||
if (file.isFile()) {
|
||||
size += entrySize(file.length());
|
||||
} else {
|
||||
size += tarSize(file);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Empty folder header
|
||||
return TarConstants.HEADER_BLOCK;
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
private static long entrySize(long fileSize) {
|
||||
long size = 0;
|
||||
size += TarConstants.HEADER_BLOCK; // Header
|
||||
size += fileSize; // File size
|
||||
|
||||
long extra = size % TarConstants.DATA_BLOCK;
|
||||
|
||||
if (extra > 0) {
|
||||
size += (TarConstants.DATA_BLOCK - extra); // pad
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
public static String trim(String s, char c) {
|
||||
StringBuffer tmp = new StringBuffer(s);
|
||||
for (int i = 0; i < tmp.length(); i++) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = tmp.length() - 1; i >= 0; i--) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
return tmp.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,400 +0,0 @@
|
||||
|
||||
package org.kivy.android;
|
||||
|
||||
import java.net.Socket;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import android.os.SystemClock;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import android.app.*;
|
||||
import android.content.*;
|
||||
import android.view.*;
|
||||
import android.view.SurfaceView;
|
||||
import android.app.Activity;
|
||||
import android.content.Intent;
|
||||
import android.util.Log;
|
||||
import android.widget.Toast;
|
||||
import android.os.AsyncTask;
|
||||
import android.os.Bundle;
|
||||
import android.os.PowerManager;
|
||||
import android.graphics.PixelFormat;
|
||||
import android.view.SurfaceHolder;
|
||||
import android.content.Context;
|
||||
import android.content.pm.ActivityInfo;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.pm.ApplicationInfo;
|
||||
import android.content.Intent;
|
||||
import android.widget.ImageView;
|
||||
import java.io.InputStream;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.Color;
|
||||
|
||||
import android.widget.AbsoluteLayout;
|
||||
|
||||
import android.webkit.WebViewClient;
|
||||
import android.webkit.WebView;
|
||||
|
||||
import org.kivy.android.PythonUtil;
|
||||
|
||||
import org.renpy.android.ResourceManager;
|
||||
import org.renpy.android.AssetExtract;
|
||||
|
||||
public class PythonActivity extends Activity {
|
||||
// This activity is modified from a mixture of the SDLActivity and
|
||||
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
|
||||
// specifics.
|
||||
|
||||
private static final String TAG = "PythonActivity";
|
||||
|
||||
public static PythonActivity mActivity = null;
|
||||
|
||||
/** If shared libraries (e.g. the native application) could not be loaded. */
|
||||
public static boolean mBrokenLibraries;
|
||||
|
||||
protected static Thread mPythonThread;
|
||||
|
||||
private ResourceManager resourceManager = null;
|
||||
private Bundle mMetaData = null;
|
||||
private PowerManager.WakeLock mWakeLock = null;
|
||||
|
||||
public String getAppRoot() {
|
||||
String app_root = getFilesDir().getAbsolutePath() + "/app";
|
||||
return app_root;
|
||||
}
|
||||
|
||||
public static void initialize() {
|
||||
// The static nature of the singleton and Android quirkiness force us to initialize everything here
|
||||
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
|
||||
mBrokenLibraries = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
Log.v(TAG, "My oncreate running");
|
||||
resourceManager = new ResourceManager(this);
|
||||
|
||||
Log.v(TAG, "Ready to unpack");
|
||||
File app_root_file = new File(getAppRoot());
|
||||
unpackData("private", app_root_file);
|
||||
|
||||
Log.v(TAG, "About to do super onCreate");
|
||||
super.onCreate(savedInstanceState);
|
||||
Log.v(TAG, "Did super onCreate");
|
||||
|
||||
this.mActivity = this;
|
||||
//this.showLoadingScreen();
|
||||
Log.v("Python", "Device: " + android.os.Build.DEVICE);
|
||||
Log.v("Python", "Model: " + android.os.Build.MODEL);
|
||||
|
||||
//Log.v(TAG, "Ready to unpack");
|
||||
//new UnpackFilesTask().execute(getAppRoot());
|
||||
|
||||
PythonActivity.initialize();
|
||||
|
||||
// Load shared libraries
|
||||
String errorMsgBrokenLib = "";
|
||||
try {
|
||||
loadLibraries();
|
||||
} catch(UnsatisfiedLinkError e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
} catch(Exception e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
}
|
||||
|
||||
if (mBrokenLibraries)
|
||||
{
|
||||
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
|
||||
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
|
||||
+ System.getProperty("line.separator")
|
||||
+ System.getProperty("line.separator")
|
||||
+ "Error: " + errorMsgBrokenLib);
|
||||
dlgAlert.setTitle("Python Error");
|
||||
dlgAlert.setPositiveButton("Exit",
|
||||
new DialogInterface.OnClickListener() {
|
||||
@Override
|
||||
public void onClick(DialogInterface dialog,int id) {
|
||||
// if this button is clicked, close current activity
|
||||
PythonActivity.mActivity.finish();
|
||||
}
|
||||
});
|
||||
dlgAlert.setCancelable(false);
|
||||
dlgAlert.create().show();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Set up the Python environment
|
||||
String app_root_dir = getAppRoot();
|
||||
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
|
||||
|
||||
Log.v(TAG, "Setting env vars for start.c and Python to use");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
|
||||
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
|
||||
|
||||
try {
|
||||
Log.v(TAG, "Access to our meta-data...");
|
||||
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
|
||||
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
|
||||
|
||||
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
|
||||
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
|
||||
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
|
||||
mActivity.mWakeLock.acquire();
|
||||
}
|
||||
} catch (PackageManager.NameNotFoundException e) {
|
||||
}
|
||||
|
||||
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
|
||||
PythonActivity.mPythonThread = pythonThread;
|
||||
pythonThread.start();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDestroy() {
|
||||
Log.i("Destroy", "end of app");
|
||||
super.onDestroy();
|
||||
|
||||
// make sure all child threads (python_thread) are stopped
|
||||
android.os.Process.killProcess(android.os.Process.myPid());
|
||||
}
|
||||
|
||||
public void loadLibraries() {
|
||||
String app_root = new String(getAppRoot());
|
||||
File app_root_file = new File(app_root);
|
||||
PythonUtil.loadLibraries(app_root_file);
|
||||
}
|
||||
|
||||
public void recursiveDelete(File f) {
|
||||
if (f.isDirectory()) {
|
||||
for (File r : f.listFiles()) {
|
||||
recursiveDelete(r);
|
||||
}
|
||||
}
|
||||
f.delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Show an error using a toast. (Only makes sense from non-UI
|
||||
* threads.)
|
||||
*/
|
||||
public void toastError(final String msg) {
|
||||
|
||||
final Activity thisActivity = this;
|
||||
|
||||
runOnUiThread(new Runnable () {
|
||||
public void run() {
|
||||
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
|
||||
}
|
||||
});
|
||||
|
||||
// Wait to show the error.
|
||||
synchronized (this) {
|
||||
try {
|
||||
this.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void unpackData(final String resource, File target) {
|
||||
|
||||
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
|
||||
|
||||
// The version of data in memory and on disk.
|
||||
String data_version = resourceManager.getString(resource + "_version");
|
||||
String disk_version = null;
|
||||
|
||||
Log.v(TAG, "Data version is " + data_version);
|
||||
|
||||
// If no version, no unpacking is necessary.
|
||||
if (data_version == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check the current disk version, if any.
|
||||
String filesDir = target.getAbsolutePath();
|
||||
String disk_version_fn = filesDir + "/" + resource + ".version";
|
||||
|
||||
try {
|
||||
byte buf[] = new byte[64];
|
||||
InputStream is = new FileInputStream(disk_version_fn);
|
||||
int len = is.read(buf);
|
||||
disk_version = new String(buf, 0, len);
|
||||
is.close();
|
||||
} catch (Exception e) {
|
||||
disk_version = "";
|
||||
}
|
||||
|
||||
// If the disk data is out of date, extract it and write the
|
||||
// version file.
|
||||
// if (! data_version.equals(disk_version)) {
|
||||
if (! data_version.equals(disk_version)) {
|
||||
Log.v(TAG, "Extracting " + resource + " assets.");
|
||||
|
||||
recursiveDelete(target);
|
||||
target.mkdirs();
|
||||
|
||||
AssetExtract ae = new AssetExtract(this);
|
||||
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
|
||||
toastError("Could not extract " + resource + " data.");
|
||||
}
|
||||
|
||||
try {
|
||||
// Write .nomedia.
|
||||
new File(target, ".nomedia").createNewFile();
|
||||
|
||||
// Write version file.
|
||||
FileOutputStream os = new FileOutputStream(disk_version_fn);
|
||||
os.write(data_version.getBytes());
|
||||
os.close();
|
||||
} catch (Exception e) {
|
||||
Log.w("python", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long lastBackClick = SystemClock.elapsedRealtime();
|
||||
@Override
|
||||
public boolean onKeyDown(int keyCode, KeyEvent event) {
|
||||
// If it wasn't the Back key or there's no web page history, bubble up to the default
|
||||
// system behavior (probably exit the activity)
|
||||
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
Toast.makeText(this, "Click again to close the app",
|
||||
Toast.LENGTH_LONG).show();
|
||||
return true;
|
||||
}
|
||||
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
return super.onKeyDown(keyCode, event);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onNewIntent
|
||||
//
|
||||
|
||||
public interface NewIntentListener {
|
||||
void onNewIntent(Intent intent);
|
||||
}
|
||||
|
||||
private List<NewIntentListener> newIntentListeners = null;
|
||||
|
||||
public void registerNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
|
||||
this.newIntentListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.newIntentListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onNewIntent(Intent intent) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.newIntentListeners ) {
|
||||
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
|
||||
while ( iterator.hasNext() ) {
|
||||
(iterator.next()).onNewIntent(intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onActivityResult
|
||||
//
|
||||
|
||||
public interface ActivityResultListener {
|
||||
void onActivityResult(int requestCode, int resultCode, Intent data);
|
||||
}
|
||||
|
||||
private List<ActivityResultListener> activityResultListeners = null;
|
||||
|
||||
public void registerActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
|
||||
this.activityResultListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.activityResultListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.activityResultListeners ) {
|
||||
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
|
||||
while ( iterator.hasNext() )
|
||||
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
|
||||
}
|
||||
}
|
||||
|
||||
public static void start_service(String serviceTitle, String serviceDescription,
|
||||
String pythonServiceArgument) {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
|
||||
String filesDirectory = argument;
|
||||
String app_root_dir = PythonActivity.mActivity.getAppRoot();
|
||||
serviceIntent.putExtra("androidPrivate", argument);
|
||||
serviceIntent.putExtra("androidArgument", app_root_dir);
|
||||
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
|
||||
serviceIntent.putExtra("pythonName", "python");
|
||||
serviceIntent.putExtra("pythonHome", app_root_dir);
|
||||
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
serviceIntent.putExtra("serviceTitle", serviceTitle);
|
||||
serviceIntent.putExtra("serviceDescription", serviceDescription);
|
||||
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
PythonActivity.mActivity.startService(serviceIntent);
|
||||
}
|
||||
|
||||
public static void stop_service() {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
PythonActivity.mActivity.stopService(serviceIntent);
|
||||
}
|
||||
|
||||
|
||||
public static native void nativeSetEnv(String j_name, String j_value);
|
||||
public static native int nativeInit(Object arguments);
|
||||
|
||||
}
|
||||
|
||||
|
||||
class PythonMain implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
PythonActivity.nativeInit(new String[0]);
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonEvent {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
private final Condition cond = lock.newCondition();
|
||||
private boolean flag = false;
|
||||
|
||||
public void set() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = true;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void wait_() throws InterruptedException {
|
||||
lock.lock();
|
||||
try {
|
||||
while (!flag) {
|
||||
cond.await();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = false;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonLock {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
|
||||
public void acquire() {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
package org.renpy.android;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.IntentFilter;
|
||||
import android.hardware.Sensor;
|
||||
import android.hardware.SensorEvent;
|
||||
import android.hardware.SensorEventListener;
|
||||
import android.hardware.SensorManager;
|
||||
import android.net.ConnectivityManager;
|
||||
import android.net.NetworkInfo;
|
||||
import android.net.wifi.ScanResult;
|
||||
import android.net.wifi.WifiManager;
|
||||
import android.os.Vibrator;
|
||||
import android.view.View;
|
||||
|
||||
/**
|
||||
* Methods that are expected to be called via JNI, to access the device's
|
||||
* non-screen hardware. (For example, the vibration and accelerometer.)
|
||||
*/
|
||||
public class Hardware {
|
||||
|
||||
// The context.
|
||||
static Context context;
|
||||
static View view;
|
||||
|
||||
/**
|
||||
* Vibrate for s seconds.
|
||||
*/
|
||||
public static void vibrate(double s) {
|
||||
Vibrator v = (Vibrator) context
|
||||
.getSystemService(Context.VIBRATOR_SERVICE);
|
||||
if (v != null) {
|
||||
v.vibrate((int) (1000 * s));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an Overview of all Hardware Sensors of an Android Device
|
||||
*/
|
||||
public static String getHardwareSensors() {
|
||||
SensorManager sm = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
|
||||
|
||||
if (allSensors != null) {
|
||||
String resultString = "";
|
||||
for (Sensor s : allSensors) {
|
||||
resultString += String.format("Name=" + s.getName());
|
||||
resultString += String.format(",Vendor=" + s.getVendor());
|
||||
resultString += String.format(",Version=" + s.getVersion());
|
||||
resultString += String.format(",MaximumRange="
|
||||
+ s.getMaximumRange());
|
||||
// XXX MinDelay is not in the 2.2
|
||||
// resultString += String.format(",MinDelay=" +
|
||||
// s.getMinDelay());
|
||||
resultString += String.format(",Power=" + s.getPower());
|
||||
resultString += String.format(",Type=" + s.getType() + "\n");
|
||||
}
|
||||
return resultString;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
|
||||
* Magnetic Field Sensors
|
||||
*/
|
||||
public static class generic3AxisSensor implements SensorEventListener {
|
||||
private final SensorManager sSensorManager;
|
||||
private final Sensor sSensor;
|
||||
private final int sSensorType;
|
||||
SensorEvent sSensorEvent;
|
||||
|
||||
public generic3AxisSensor(int sensorType) {
|
||||
sSensorType = sensorType;
|
||||
sSensorManager = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
sSensor = sSensorManager.getDefaultSensor(sSensorType);
|
||||
}
|
||||
|
||||
public void onAccuracyChanged(Sensor sensor, int accuracy) {
|
||||
}
|
||||
|
||||
public void onSensorChanged(SensorEvent event) {
|
||||
sSensorEvent = event;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable the Sensor by registering/unregistering
|
||||
*/
|
||||
public void changeStatus(boolean enable) {
|
||||
if (enable) {
|
||||
sSensorManager.registerListener(this, sSensor,
|
||||
SensorManager.SENSOR_DELAY_NORMAL);
|
||||
} else {
|
||||
sSensorManager.unregisterListener(this, sSensor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the Sensor
|
||||
*/
|
||||
public float[] readSensor() {
|
||||
if (sSensorEvent != null) {
|
||||
return sSensorEvent.values;
|
||||
} else {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static generic3AxisSensor accelerometerSensor = null;
|
||||
public static generic3AxisSensor orientationSensor = null;
|
||||
public static generic3AxisSensor magneticFieldSensor = null;
|
||||
|
||||
/**
|
||||
* functions for backward compatibility reasons
|
||||
*/
|
||||
|
||||
public static void accelerometerEnable(boolean enable) {
|
||||
if (accelerometerSensor == null)
|
||||
accelerometerSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_ACCELEROMETER);
|
||||
accelerometerSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] accelerometerReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (accelerometerSensor == null)
|
||||
return rv;
|
||||
return (float[]) accelerometerSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void orientationSensorEnable(boolean enable) {
|
||||
if (orientationSensor == null)
|
||||
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
|
||||
orientationSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] orientationSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (orientationSensor == null)
|
||||
return rv;
|
||||
return (float[]) orientationSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void magneticFieldSensorEnable(boolean enable) {
|
||||
if (magneticFieldSensor == null)
|
||||
magneticFieldSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_MAGNETIC_FIELD);
|
||||
magneticFieldSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] magneticFieldSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (magneticFieldSensor == null)
|
||||
return rv;
|
||||
return (float[]) magneticFieldSensor.readSensor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan WiFi networks
|
||||
*/
|
||||
static List<ScanResult> latestResult;
|
||||
|
||||
public static void enableWifiScanner() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
|
||||
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
|
||||
// occurs
|
||||
WifiManager w = (WifiManager) c
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
latestResult = w.getScanResults(); // Returns a <list> of
|
||||
// scanResults
|
||||
}
|
||||
|
||||
}, i);
|
||||
|
||||
}
|
||||
|
||||
public static String scanWifi() {
|
||||
|
||||
// Now you can call this and it should execute the broadcastReceiver's
|
||||
// onReceive()
|
||||
WifiManager wm = (WifiManager) context
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
boolean a = wm.startScan();
|
||||
|
||||
if (latestResult != null) {
|
||||
|
||||
String latestResultString = "";
|
||||
for (ScanResult result : latestResult) {
|
||||
latestResultString += String.format("%s\t%s\t%d\n",
|
||||
result.SSID, result.BSSID, result.level);
|
||||
}
|
||||
|
||||
return latestResultString;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* network state
|
||||
*/
|
||||
|
||||
public static boolean network_state = false;
|
||||
|
||||
/**
|
||||
* Check network state directly
|
||||
*
|
||||
* (only one connection can be active at a given moment, detects all network
|
||||
* type)
|
||||
*
|
||||
*/
|
||||
public static boolean checkNetwork() {
|
||||
boolean state = false;
|
||||
final ConnectivityManager conMgr = (ConnectivityManager) context
|
||||
.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||
|
||||
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
|
||||
if (activeNetwork != null && activeNetwork.isConnected()) {
|
||||
state = true;
|
||||
} else {
|
||||
state = false;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* To recieve network state changes
|
||||
*/
|
||||
public static void registerNetworkCheck() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
network_state = checkNetwork();
|
||||
}
|
||||
|
||||
}, i);
|
||||
}
|
||||
|
||||
}
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -1,93 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="{{ args.package }}"
|
||||
android:versionCode="{{ args.numeric_version }}"
|
||||
android:versionName="{{ args.version }}"
|
||||
android:installLocation="auto">
|
||||
|
||||
<supports-screens
|
||||
android:smallScreens="true"
|
||||
android:normalScreens="true"
|
||||
android:largeScreens="true"
|
||||
android:anyDensity="true"
|
||||
{% if args.min_sdk_version >= 9 %}
|
||||
android:xlargeScreens="true"
|
||||
{% endif %}
|
||||
/>
|
||||
|
||||
<!-- Android 2.3.3 -->
|
||||
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
|
||||
|
||||
<!-- Set permissions -->
|
||||
{% for perm in args.permissions %}
|
||||
{% if '.' in perm %}
|
||||
<uses-permission android:name="{{ perm }}" />
|
||||
{% else %}
|
||||
<uses-permission android:name="android.permission.{{ perm }}" />
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% if args.wakelock %}
|
||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||
{% endif %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<uses-permission android:name="com.android.vending.BILLING" />
|
||||
{% endif %}
|
||||
|
||||
<!-- Create a Java class extending SDLActivity and place it in a
|
||||
directory under src matching the package, e.g.
|
||||
src/com/gamemaker/game/MyGame.java
|
||||
|
||||
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
|
||||
in the XML below.
|
||||
|
||||
An example Java class can be found in README-android.txt
|
||||
-->
|
||||
<application android:label="@string/app_name"
|
||||
android:icon="@drawable/icon"
|
||||
android:allowBackup="true"
|
||||
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
|
||||
android:hardwareAccelerated="true" >
|
||||
|
||||
{% for m in args.meta_data %}
|
||||
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
|
||||
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
|
||||
|
||||
<activity android:name="org.kivy.android.PythonActivity"
|
||||
android:label="@string/app_name"
|
||||
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
|
||||
>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
{%- if args.intent_filters -%}
|
||||
{{- args.intent_filters -}}
|
||||
{%- endif -%}
|
||||
</activity>
|
||||
|
||||
{% if service %}
|
||||
<service android:name="org.kivy.android.PythonService"
|
||||
android:process=":pythonservice" />
|
||||
{% endif %}
|
||||
{% for name in service_names %}
|
||||
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
|
||||
android:process=":service_{{ name }}" />
|
||||
{% endfor %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<service android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbilling" />
|
||||
<receiver android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbillingreceiver">
|
||||
<intent-filter>
|
||||
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
|
||||
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
|
||||
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
{% endif %}
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -1,74 +0,0 @@
|
||||
package {{ args.package }};
|
||||
|
||||
import android.os.Binder;
|
||||
import android.os.IBinder;
|
||||
import android.content.Intent;
|
||||
import android.content.Context;
|
||||
import org.kivy.android.PythonService;
|
||||
|
||||
public class Service{{ name|capitalize }} extends PythonService {
|
||||
/**
|
||||
* Binder given to clients
|
||||
*/
|
||||
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
|
||||
|
||||
{% if sticky %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public int getStartType() {
|
||||
return START_STICKY;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% if foreground %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean getStartForeground() {
|
||||
return true;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
public static void start(Context ctx, String pythonServiceArgument) {
|
||||
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
intent.putExtra("androidPrivate", argument);
|
||||
intent.putExtra("androidArgument", argument);
|
||||
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
|
||||
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
|
||||
intent.putExtra("serviceDescription", "");
|
||||
intent.putExtra("pythonName", "{{ name }}");
|
||||
intent.putExtra("pythonHome", argument);
|
||||
intent.putExtra("androidUnpack", argument);
|
||||
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
|
||||
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
ctx.startService(intent);
|
||||
}
|
||||
|
||||
public static void stop(Context ctx) {
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
ctx.stopService(intent);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public IBinder onBind(Intent intent) {
|
||||
return mBinder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Class used for the client Binder. Because we know this service always
|
||||
* runs in the same process as its clients, we don't need to deal with IPC.
|
||||
*/
|
||||
public class Service{{ name|capitalize }}Binder extends Binder {
|
||||
Service{{ name|capitalize }} getService() {
|
||||
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
|
||||
return Service{{ name|capitalize }}.this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">{{ args.name }}</string>
|
||||
<string name="private_version">{{ private_version }}</string>
|
||||
</resources>
|
||||
@@ -1,68 +0,0 @@
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from jnius import PythonJavaClass, autoclass, java_method
|
||||
JAVA_NAMESPACE = 'org.kivy.android'
|
||||
JNI_NAMESPACE = 'org/kivy/android'
|
||||
|
||||
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
|
||||
|
||||
_callbacks = {
|
||||
'on_new_intent': [],
|
||||
'on_activity_result': [],
|
||||
}
|
||||
|
||||
|
||||
class NewIntentListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback, **kwargs):
|
||||
super(NewIntentListener, self).__init__(**kwargs)
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(Landroid/content/Intent;)V')
|
||||
def onNewIntent(self, intent):
|
||||
self.callback(intent)
|
||||
|
||||
|
||||
class ActivityResultListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback):
|
||||
super(ActivityResultListener, self).__init__()
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(IILandroid/content/Intent;)V')
|
||||
def onActivityResult(self, requestCode, resultCode, intent):
|
||||
self.callback(requestCode, resultCode, intent)
|
||||
|
||||
|
||||
def bind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
elif event == 'on_new_intent':
|
||||
listener = NewIntentListener(callback)
|
||||
_activity.registerNewIntentListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
elif event == 'on_activity_result':
|
||||
listener = ActivityResultListener(callback)
|
||||
_activity.registerActivityResultListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
|
||||
|
||||
def unbind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
else:
|
||||
for listener in _callbacks[event][:]:
|
||||
if listener.callback == callback:
|
||||
_callbacks[event].remove(listener)
|
||||
if event == 'on_new_intent':
|
||||
_activity.unregisterNewIntentListener(listener)
|
||||
elif event == 'on_activity_result':
|
||||
_activity.unregisterActivityResultListener(listener)
|
||||
|
||||
|
||||
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
|
||||
@@ -1,49 +0,0 @@
|
||||
from os.path import join, dirname, abspath
|
||||
from pythonforandroid.toolchain import Bootstrap
|
||||
from setuptools import setup, find_packages
|
||||
from bootstrap import LBRYServiceBootstrap
|
||||
|
||||
|
||||
Bootstrap.bootstraps = {
|
||||
'lbry-service': LBRYServiceBootstrap()
|
||||
}
|
||||
|
||||
|
||||
setup(
|
||||
name='lbryservice',
|
||||
version='0.1',
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
description="Android Service for LBRY Network.",
|
||||
license='MIT',
|
||||
python_requires='>=3.7',
|
||||
packages=find_packages(),
|
||||
package_data={'service': ['*.py']},
|
||||
options={
|
||||
'apk': {
|
||||
'dist_name': 'lbry-service',
|
||||
'bootstrap': 'lbry-service',
|
||||
'package': 'io.lbry.service',
|
||||
'permissions': ['INTERNET'],
|
||||
'requirements': ','.join([
|
||||
# needed by aiohttp
|
||||
'multidict', 'yarl', 'async_timeout', 'chardet',
|
||||
# minimum needed by torba:
|
||||
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
|
||||
abspath(join(dirname(__file__), '..', '..', 'torba')),
|
||||
# minimum needed by lbrynet
|
||||
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
|
||||
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
|
||||
abspath(join(dirname(__file__), '..')),
|
||||
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
|
||||
]),
|
||||
'android-api': '26',
|
||||
'ndk-api': '21',
|
||||
'ndk-version': 'r17c',
|
||||
'arch': 'armeabi-v7a',
|
||||
'sdk-dir': '/home/lex/projects/android',
|
||||
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
|
||||
}
|
||||
}
|
||||
)
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.31.0rc3"
|
||||
__version__ = "0.31.0rc4"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -77,7 +77,7 @@ class BlobFile:
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.result()
|
||||
error = finished.exception()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
@@ -86,7 +86,7 @@ class BlobFile:
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
@@ -96,24 +96,24 @@ class BlobFile:
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer):
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(writer.verified_bytes)
|
||||
write_handle.write(verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if os.path.exists(self.file_path):
|
||||
|
||||
@@ -18,7 +18,6 @@ class HashBlobWriter:
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.buffer is not None:
|
||||
@@ -46,7 +45,7 @@ class HashBlobWriter:
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidDataError(
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
@@ -55,15 +54,12 @@ class HashBlobWriter:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.buffer is not None:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
@@ -19,15 +20,15 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
def data_received(self, data: bytes):
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
|
||||
@@ -36,60 +37,74 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
return
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
|
||||
def data_received(self, data):
|
||||
try:
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
self.transport.write(request.serialize())
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return False, False
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected")
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return False, True
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch")
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length")
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.info(msg)
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
@@ -104,9 +119,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
@@ -117,52 +129,49 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
try:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
finally:
|
||||
await protocol.close()
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
@@ -18,115 +18,94 @@ def drain_into(a: list, b: list):
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
class BlobDownloader:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Set['KademliaPeer'] = set()
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
async def _request_blob():
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
try:
|
||||
success, keep_connection = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout
|
||||
)
|
||||
finally:
|
||||
if peer in self.active_connections:
|
||||
self.active_connections.pop(peer)
|
||||
if not keep_connection and peer not in self.ignored:
|
||||
self.ignored.add(peer)
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
elif keep_connection:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
if success:
|
||||
self.scores[peer] = self.scores.get(peer, 0) + 2
|
||||
else:
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1
|
||||
return self.loop.create_task(_request_blob())
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
new_peers = await self.peer_queue.get()
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
try:
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
await asyncio.wait(tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
drain_tasks(tasks)
|
||||
raise
|
||||
finally:
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(await self.peer_queue.get())
|
||||
for peer in batch:
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
self.active_connections[peer] = self.request_blob_from_peer(blob, peer)
|
||||
await self.new_peer_or_finished(blob)
|
||||
to_re_add = list(set(filter(lambda peer: peer not in self.ignored, batch)))
|
||||
to_re_add.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
if to_re_add:
|
||||
self.peer_queue.put_nowait(to_re_add)
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
await blob.close()
|
||||
log.info("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
raise
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
@@ -146,32 +146,26 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
break
|
||||
return None, response_msg
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
|
||||
@@ -57,8 +57,13 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
@@ -72,7 +77,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(data)
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
|
||||
+63
-64
@@ -16,29 +16,11 @@ log = logging.getLogger(__name__)
|
||||
NOT_SET = type(str('NOT_SET'), (object,), {})
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
API_ADDRESS = 'lbryapi'
|
||||
APP_NAME = 'LBRY'
|
||||
BLOBFILES_DIR = 'blobfiles'
|
||||
CRYPTSD_FILE_EXTENSION = '.cryptsd'
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
|
||||
LOG_FILE_NAME = 'lbrynet.log'
|
||||
LOG_POST_URL = 'https://lbry.io/log-upload'
|
||||
MAX_BLOB_REQUEST_SIZE = 64 * KB
|
||||
MAX_HANDSHAKE_SIZE = 64 * KB
|
||||
MAX_REQUEST_SIZE = 64 * KB
|
||||
MAX_RESPONSE_INFO_SIZE = 64 * KB
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
@@ -471,7 +453,7 @@ class CLIConfig(BaseConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
@@ -482,60 +464,77 @@ class Config(CLIConfig):
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 20.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
) # TODO: use this
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# claims set to expire within this many blocks will be
|
||||
# automatically renewed after startup (if set to 0, renews
|
||||
# will not be made automatically)
|
||||
auto_renew_claim_height_delta = Integer("", 0)
|
||||
cache_time = Integer("", 150)
|
||||
data_rate = Float("points/megabyte", .0001)
|
||||
delete_blobs_on_remove = Toggle("", True)
|
||||
dht_node_port = Integer("", 4444)
|
||||
download_timeout = Float("", 30.0)
|
||||
blob_download_timeout = Float("", 20.0)
|
||||
peer_connect_timeout = Float("", 3.0)
|
||||
node_rpc_timeout = Float("", constants.rpc_timeout)
|
||||
is_generous_host = Toggle("", True)
|
||||
announce_head_blobs_only = Toggle("", True)
|
||||
concurrent_announcers = Integer("", 10)
|
||||
known_dht_nodes = Servers("", [
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
max_connections_per_stream = Integer("", 5)
|
||||
seek_head_blob_first = Toggle("", True)
|
||||
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
|
||||
min_info_rate = Float("points/1000 infos", .02)
|
||||
min_valuable_hash_rate = Float("points/1000 infos", .05)
|
||||
min_valuable_info_rate = Float("points/1000 infos", .05)
|
||||
peer_port = Integer("", 3333)
|
||||
pointtrader_server = String("", 'http://127.0.0.1:2424')
|
||||
reflector_port = Integer("", 5566)
|
||||
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
reflect_uploads = Toggle("", True)
|
||||
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
|
||||
reflector_servers = Servers("", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
|
||||
sd_download_timeout = Integer("", 3)
|
||||
peer_search_timeout = Integer("", 60)
|
||||
use_upnp = Toggle("", True)
|
||||
use_keyring = Toggle("", False)
|
||||
blockchain_name = String("", 'lbrycrd_main')
|
||||
lbryum_servers = Servers("", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import time
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
@@ -9,7 +10,8 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage',
|
||||
time_getter: typing.Callable[[], float] = time.time):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
@@ -17,8 +19,11 @@ class BlobAnnouncer:
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.running = False
|
||||
self.announce_queue: typing.List[str] = []
|
||||
self.time_getter = time_getter
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
@@ -39,7 +44,7 @@ class BlobAnnouncer:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced, self.loop.time())
|
||||
await self.storage.update_last_announced_blobs(announced, self.time_getter())
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
|
||||
+55
-32
@@ -1,9 +1,9 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import socket
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -20,10 +20,10 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -79,10 +79,9 @@ class Node:
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.info("Store to %i peers", len(peers))
|
||||
log.info(peers)
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
@@ -116,39 +115,50 @@ class Node:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
known_node_addresses = known_node_addresses or []
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
info = await self.loop.getaddrinfo(
|
||||
host, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
if (info[0][4][0], port) not in known_node_addresses:
|
||||
known_node_addresses.append((info[0][4][0], port))
|
||||
futs = []
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
futs.append(peer.ping())
|
||||
if futs:
|
||||
await asyncio.wait(futs, loop=self.loop)
|
||||
address = await resolve_host(host)
|
||||
if (address, port) not in known_node_addresses:
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
if known_node_addresses:
|
||||
while not self.protocol.routing_table.get_peers():
|
||||
success = False
|
||||
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
self.joined.set()
|
||||
await peer.ping()
|
||||
success = True
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
|
||||
round(self.protocol.rpc_timeout, 2))
|
||||
if success:
|
||||
break
|
||||
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
|
||||
# in the routing table with good peers who are near us
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
@@ -214,13 +224,26 @@ class Node:
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
|
||||
async with self.peer_search_junction(node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
log.info("peer search: %s", peers)
|
||||
accumulated.extend(peers)
|
||||
log.info("junction done")
|
||||
log.info("context done")
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
try:
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait(peers)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue()
|
||||
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
+33
-54
@@ -30,37 +30,30 @@ class PeerManager:
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
self._lock = asyncio.Lock(loop=loop)
|
||||
|
||||
async def report_failure(self, address: str, udp_port: int):
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
async def report_last_sent(self, address: str, udp_port: int):
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
async def report_last_replied(self, address: str, udp_port: int):
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
async def report_last_requested(self, address: str, udp_port: int):
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
async def clear_token(self, node_id: bytes):
|
||||
async with self._lock:
|
||||
self._node_tokens.pop(node_id, None)
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
async def update_token(self, node_id: bytes, token: bytes):
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
@@ -70,50 +63,36 @@ class PeerManager:
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
|
||||
return self._node_id_mapping.get((address, udp_port))
|
||||
|
||||
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
return self._node_id_reverse_mapping.get(node_id)
|
||||
|
||||
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
async with self._lock:
|
||||
addr_tuple = self._node_id_reverse_mapping.get(node_id)
|
||||
if addr_tuple and addr_tuple in self._node_id_mapping:
|
||||
return addr_tuple
|
||||
|
||||
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
async with self._lock:
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
async def prune(self):
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
|
||||
@@ -100,6 +100,6 @@ class AsyncGeneratorJunction:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
@@ -90,8 +92,8 @@ class IterativeFinder:
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.List['KademliaPeer'] = []
|
||||
self.contacted: typing.List[typing.Tuple[str, int]] = []
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
@@ -99,14 +101,12 @@ class IterativeFinder:
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.List[asyncio.Task] = []
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
@@ -114,9 +114,16 @@ class IterativeFinder:
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called with a lock after adding peers from an rpc result to the shortlist.
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
@@ -129,9 +136,7 @@ class IterativeFinder:
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
if not self.closest_peer:
|
||||
return True
|
||||
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
@@ -141,21 +146,18 @@ class IterativeFinder:
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
async with self.lock:
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.append(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
addr_tuple = (contact_triple[1], contact_triple[2])
|
||||
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(
|
||||
contact_triple[0], contact_triple[1], contact_triple[2]
|
||||
)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.add(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
@@ -163,13 +165,11 @@ class IterativeFinder:
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except RemoteException:
|
||||
return
|
||||
@@ -181,31 +181,35 @@ class IterativeFinder:
|
||||
"""
|
||||
|
||||
added = 0
|
||||
async with self.lock:
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.append((peer.address, peer.udp_port))
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.add((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
if t and t in self.running_probes:
|
||||
self.running_probes.remove(t)
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.append(t)
|
||||
added += 1
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
@@ -215,70 +219,41 @@ class IterativeFinder:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
if self.running:
|
||||
drain_tasks(self.running_probes)
|
||||
self.running = False
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def search(self):
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
|
||||
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
|
||||
peers = self.loop.create_task(self.iteration_queue.get())
|
||||
finished = self.loop.create_task(self.finished.wait())
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
if peers.done():
|
||||
return peers.result()
|
||||
raise StopAsyncIteration()
|
||||
except asyncio.CancelledError as error:
|
||||
err = error
|
||||
finally:
|
||||
if not finished.done() and not finished.cancelled():
|
||||
finished.cancel()
|
||||
if not peers.done() and not peers.cancelled():
|
||||
peers.cancel()
|
||||
if err:
|
||||
raise err
|
||||
|
||||
def __aiter__(self):
|
||||
self.search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
initial_results = self.get_initial_result()
|
||||
if initial_results:
|
||||
self.iteration_queue.put_nowait(initial_results)
|
||||
result = await self.next_queue_or_finished()
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
await self.aclose()
|
||||
self.loop.call_soon(self.aclose)
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
async def _aclose():
|
||||
async with self.lock:
|
||||
self.running = False
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
drain_tasks(self.tasks)
|
||||
drain_tasks(self.running_probes)
|
||||
while self.delayed_calls:
|
||||
timer = self.delayed_calls.pop()
|
||||
if timer:
|
||||
timer.cancel()
|
||||
|
||||
return asyncio.ensure_future(_aclose(), loop=self.loop)
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
@@ -295,24 +270,26 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def put_result(self, from_list: typing.List['KademliaPeer']):
|
||||
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.info("found")
|
||||
self.put_result(self.shortlist)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
log.debug("found")
|
||||
return self.put_result(self.shortlist, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
@@ -323,16 +300,10 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.info("max results")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
self.put_result(self.active, True)
|
||||
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.debug("max results")
|
||||
self.put_result(self.active, True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
@@ -366,14 +337,11 @@ class IterativeValueFinder(IterativeFinder):
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer:
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
@@ -86,7 +86,7 @@ class KademliaRPC:
|
||||
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
|
||||
peers.extend([peer.compact_address_tcp() for peer in self.protocol.data_store.get_peers_for_blob(key)])
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
@@ -180,7 +180,7 @@ class RemoteKademliaRPC:
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
@@ -218,7 +218,7 @@ class PingQueue:
|
||||
await self._protocol.add_peer(p)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
except TimeoutError:
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
while True:
|
||||
@@ -415,8 +415,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
await self.peer_manager.report_last_requested(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
@@ -457,8 +457,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
await self.peer_manager.report_last_replied(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -522,10 +522,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.routing_table.remove_peer(peer)
|
||||
raise
|
||||
@@ -575,9 +575,9 @@ class KademliaProtocol(DatagramProtocol):
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
@@ -601,7 +601,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
@@ -609,7 +609,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
await self.peer_manager.clear_token(peer.node_id)
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import sys
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -216,12 +218,19 @@ def get_argument_parser():
|
||||
return main
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
@@ -243,6 +252,8 @@ def main(argv=None):
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
log.addHandler(handler)
|
||||
logging.getLogger('torba').addHandler(handler)
|
||||
logging.getLogger('torba').setLevel(logging.INFO)
|
||||
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
@@ -5,7 +5,6 @@ import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import socket
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
import base58
|
||||
@@ -18,7 +17,6 @@ import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
@@ -65,14 +63,6 @@ async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
pass
|
||||
|
||||
|
||||
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
|
||||
info = await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
return info[0][4][0]
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -359,7 +349,7 @@ class DHTComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
@@ -376,8 +366,8 @@ class DHTComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
@@ -389,13 +379,14 @@ class DHTComponent(Component):
|
||||
asyncio.get_event_loop(),
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.dht_node_port,
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
)
|
||||
log.info("Started the dht")
|
||||
|
||||
@@ -419,7 +410,7 @@ class HashAnnouncerComponent(Component):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_announcers)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
async def stop(self):
|
||||
@@ -462,17 +453,13 @@ class StreamManagerComponent(Component):
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
|
||||
self.conf.peer_connect_timeout, [
|
||||
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
|
||||
for url, port in self.conf.reflector_servers
|
||||
], self.conf.reflector_servers
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
await self.stream_manager.stop()
|
||||
self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -492,10 +479,10 @@ class PeerProtocolServerComponent(Component):
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface='0.0.0.0')
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
@@ -508,8 +495,8 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
@@ -614,7 +601,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
|
||||
@@ -6,10 +6,10 @@ import inspect
|
||||
import typing
|
||||
import aiohttp
|
||||
import base58
|
||||
import random
|
||||
from urllib.parse import urlencode, quote
|
||||
from typing import Callable, Optional, List
|
||||
from binascii import hexlify, unhexlify
|
||||
from copy import deepcopy
|
||||
from traceback import format_exc
|
||||
from aiohttp import web
|
||||
from functools import wraps
|
||||
@@ -18,9 +18,9 @@ from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
|
||||
from lbrynet import __version__, utils
|
||||
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.blob_exchange.downloader import download_blob
|
||||
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
|
||||
from lbrynet.extras import system_info
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
@@ -29,6 +29,7 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
|
||||
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.extras.daemon.undecorated import undecorated
|
||||
from lbrynet.extras.wallet.account import Account as LBCAccount
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -1307,8 +1308,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
|
||||
[--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
|
||||
[--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
|
||||
[--claim_name=<claim_name>] [--sort=<sort_by>] [--reverse] [--comparison=<comparison>]
|
||||
[--full_status=<full_status>]
|
||||
[--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>]
|
||||
[--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>]
|
||||
[--comparison=<comparison>] [--full_status=<full_status>] [--reverse]
|
||||
|
||||
Options:
|
||||
--sd_hash=<sd_hash> : (str) get file with matching sd hash
|
||||
@@ -1321,11 +1323,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str) get file with matching claim txid
|
||||
--nout=<nout> : (int) get file with matching claim nout
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--claim_name=<claim_name> : (str) get file with matching claim name
|
||||
--sort=<sort_method> : (str) sort by any property, like 'file_name'
|
||||
or 'metadata.author'; to specify direction
|
||||
append ',asc' or ',desc'
|
||||
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
|
||||
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
|
||||
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
|
||||
--comparison=<comparison> : (str) logical comparision, (eq | ne | g | ge | l | le)
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
@@ -1344,21 +1347,24 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes,
|
||||
'total_bytes_lower_bound': (int) lower bound file size in bytes,
|
||||
'total_bytes': (int) file upper bound size in bytes,
|
||||
'written_bytes': (int) written size in bytes,
|
||||
'blobs_completed': (int) number of fully downloaded blobs,
|
||||
'blobs_in_stream': (int) total blobs on stream,
|
||||
'blobs_remaining': (int) total blobs remaining to download,
|
||||
'status': (str) downloader status
|
||||
'claim_id': (str) None if claim is not found else the claim id,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'txid': (str) None if claim is not found else the transaction id,
|
||||
'nout': (int) None if claim is not found else the transaction output index,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'metadata': (dict) None if claim is not found else the claim metadata,
|
||||
'channel_claim_id': (str) None if claim is not found or not signed,
|
||||
'channel_name': (str) None if claim is not found or not signed,
|
||||
'claim_name': (str) None if claim is not found else the claim name
|
||||
},
|
||||
]
|
||||
}
|
||||
"""
|
||||
sort = sort or 'status'
|
||||
comparison = comparison or 'eq'
|
||||
@@ -1551,44 +1557,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
timeout = timeout if timeout is not None else self.conf.download_timeout
|
||||
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise Exception("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet_manager.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
)
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(self.exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
fee_address = claim.source_fee.address
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.stream_manager.get_filtered_streams(outpoint=outpoint)
|
||||
if not existing:
|
||||
existing.extend(self.stream_manager.get_filtered_streams(claim_id=resolved['claim_id'],
|
||||
sd_hash=claim.source_hash))
|
||||
if existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
else:
|
||||
stream = await self.stream_manager.download_stream_from_claim(
|
||||
self.dht_node, self.conf.download_dir, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
stream = await self.stream_manager.download_stream_from_uri(
|
||||
uri, self.exchange_rate_manager, file_name, timeout
|
||||
)
|
||||
if stream:
|
||||
return stream.as_dict()
|
||||
raise DownloadSDTimeout(resolved['value']['stream']['source']['source'])
|
||||
raise DownloadSDTimeout(uri)
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_set_status(self, status, **kwargs):
|
||||
@@ -1618,11 +1592,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not streams:
|
||||
raise Exception(f'Unable to find a file for {kwargs}')
|
||||
stream = streams[0]
|
||||
if status == 'start' and not stream.running and not stream.finished:
|
||||
stream.downloader.download(self.dht_node)
|
||||
if status == 'start' and not stream.running:
|
||||
await self.stream_manager.start_stream(stream)
|
||||
msg = "Resumed download"
|
||||
elif status == 'stop' and stream.running:
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
msg = "Stopped download"
|
||||
else:
|
||||
msg = (
|
||||
@@ -1829,7 +1803,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def jsonrpc_publish(
|
||||
self, name, bid, metadata=None, file_path=None, fee=None, title=None,
|
||||
description=None, author=None, language=None, license=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None,
|
||||
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
|
||||
claim_address=None, change_address=None):
|
||||
"""
|
||||
@@ -1853,7 +1827,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
|
||||
[--description=<description>] [--author=<author>] [--language=<language>]
|
||||
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>]
|
||||
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
|
||||
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
|
||||
[--claim_address=<claim_address>] [--change_address=<change_address>]
|
||||
@@ -1889,7 +1863,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--thumbnail=<thumbnail> : (str) thumbnail url
|
||||
--preview=<preview> : (str) preview url
|
||||
--nsfw=<nsfw> : (bool) whether the content is nsfw
|
||||
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
|
||||
for channel claim being in the wallet. This allows
|
||||
@@ -1913,7 +1886,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
try:
|
||||
parse_lbry_uri(name)
|
||||
parsed = parse_lbry_uri(name)
|
||||
if parsed.name != name:
|
||||
raise Exception("Name given to publish has invalid characters")
|
||||
except (TypeError, URIParseError):
|
||||
raise Exception("Invalid name given to publish")
|
||||
|
||||
@@ -1929,9 +1904,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
account = self.get_account_or_default(account_id)
|
||||
|
||||
available = await account.get_balance()
|
||||
existing_claims = []
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) > 1:
|
||||
log.warning("found %i claims for the name %s", len(existing_claims), name)
|
||||
if amount >= available:
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) == 1:
|
||||
available += existing_claims[0].get_estimator(self.ledger).effective_amount
|
||||
if amount >= available:
|
||||
@@ -1990,39 +1966,64 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
}
|
||||
|
||||
# this will be used to verify the format with lbrynet.schema
|
||||
claim_copy = deepcopy(claim_dict)
|
||||
if sources is not None:
|
||||
claim_dict['stream']['source'] = sources
|
||||
claim_copy['stream']['source'] = sources
|
||||
elif file_path is not None:
|
||||
sd_to_delete = None
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("invalid file path to publish")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
if existing_claims:
|
||||
sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source']
|
||||
|
||||
# since the file hasn't yet been made into a stream, we don't have
|
||||
# a valid Source for the claim when validating the format, we'll use a fake one
|
||||
claim_copy['stream']['source'] = {
|
||||
claim_dict['stream']['source'] = {
|
||||
'version': '_0_0_1',
|
||||
'sourceType': 'lbry_sd_hash',
|
||||
'source': '0' * 96,
|
||||
'contentType': ''
|
||||
}
|
||||
elif not existing_claims:
|
||||
raise Exception("no previous stream to update")
|
||||
else:
|
||||
# there is no existing source to use, and a file was not provided to make a new one
|
||||
raise Exception("no source provided to publish")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
try:
|
||||
ClaimDict.load_dict(claim_copy)
|
||||
claim_dict = ClaimDict.load_dict(claim_dict).claim_dict
|
||||
# the metadata to use in the claim can be serialized by lbrynet.schema
|
||||
except DecodeError as err:
|
||||
# there was a problem with a metadata field, raise an error here rather than
|
||||
# waiting to find out when we go to publish the claim (after having made the stream)
|
||||
raise Exception(f"invalid publish metadata: {err}")
|
||||
|
||||
certificate = None
|
||||
if channel_id or channel_name:
|
||||
certificate = await self.get_channel_or_error(
|
||||
self.get_accounts_or_all(channel_account_id), channel_id, channel_name
|
||||
)
|
||||
|
||||
if file_path:
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
await self.storage.save_published_file(stream.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
|
||||
if sd_to_delete:
|
||||
stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete)
|
||||
stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete)
|
||||
if stream_to_delete:
|
||||
await self.stream_manager.delete_stream(stream_to_delete, delete_file=False)
|
||||
log.info("updating claim to stream generated from %s, deleted previous stream %s",
|
||||
sd_to_delete[:8], file_path)
|
||||
else:
|
||||
log.info("previous stream %s from claim was not saved locally, nothing to delete",
|
||||
sd_to_delete[:8])
|
||||
else:
|
||||
log.info("generated stream from %s for claim", file_path)
|
||||
else:
|
||||
log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8])
|
||||
|
||||
sd_hash = claim_dict['stream']['source']['source']
|
||||
log.info("Publish: %s", {
|
||||
'name': name,
|
||||
'file_path': file_path,
|
||||
@@ -2033,35 +2034,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'channel_id': channel_id,
|
||||
'channel_name': channel_name
|
||||
})
|
||||
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File {file_path} not found")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
claim_dict['stream']['source'] = {}
|
||||
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
stream_hash = stream.stream_hash
|
||||
await self.storage.save_published_file(stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
else:
|
||||
if not ('source' not in claim_dict['stream'] and existing_claims):
|
||||
raise Exception("no previous stream to update")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source'])
|
||||
tx = await self.wallet_manager.claim_name(
|
||||
account, name, amount, claim_dict, certificate, claim_address
|
||||
)
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
if stream_hash:
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
nout = 0
|
||||
txo = tx.outputs[nout]
|
||||
@@ -2289,7 +2269,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
claims = await self.wallet_manager.get_claims_for_name(name) # type: dict
|
||||
sort_claim_results(claims['claims'])
|
||||
claims['claims'] = sort_claim_results(claims['claims'])
|
||||
return claims
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -2569,8 +2549,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
"""
|
||||
|
||||
downloader = BlobDownloader(asyncio.get_event_loop(), self.blob_manager, self.conf)
|
||||
blob = await downloader.get_blob(blob_hash, self.dht_node)
|
||||
blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash)
|
||||
if read:
|
||||
with open(blob.file_path, 'rb') as handle:
|
||||
return handle.read().decode()
|
||||
@@ -2791,7 +2770,19 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(list) list of blobs reflected
|
||||
"""
|
||||
|
||||
raise NotImplementedError()
|
||||
server, port = kwargs.get('server'), kwargs.get('port')
|
||||
if server and port:
|
||||
port = int(port)
|
||||
else:
|
||||
server, port = random.choice(self.conf.reflector_servers)
|
||||
reflected = await asyncio.gather(*[
|
||||
stream.upload_to_reflector(server, port)
|
||||
for stream in self.stream_manager.get_filtered_streams(**kwargs)
|
||||
])
|
||||
total = []
|
||||
for reflected_for_stream in reflected:
|
||||
total.extend(reflected_for_stream)
|
||||
return total
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
async def jsonrpc_peer_ping(self, node_id, address, port):
|
||||
@@ -2817,7 +2808,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
try:
|
||||
result = await peer.ping()
|
||||
return result.decode()
|
||||
except TimeoutError:
|
||||
except asyncio.TimeoutError:
|
||||
return {'error': 'ping timeout'}
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
|
||||
@@ -5,9 +5,12 @@ import logging
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
|
||||
@@ -3,6 +3,7 @@ import sqlite3
|
||||
import typing
|
||||
import asyncio
|
||||
import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -274,8 +275,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
timestamp = int(self.loop.time())
|
||||
if self.conf.announce_head_and_sd_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
@@ -399,16 +400,15 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
return self.save_published_file(
|
||||
stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
|
||||
status="running"
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished"):
|
||||
return self.db.execute(
|
||||
"insert into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
)
|
||||
|
||||
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
|
||||
@@ -447,6 +447,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.info("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
))
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
@@ -682,7 +688,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
(sd_hash, reflector_address, time.time())
|
||||
)
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
@@ -694,5 +700,5 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
self.loop.time() - 86400
|
||||
)
|
||||
+28
-19
@@ -43,11 +43,11 @@ class StreamAssembler:
|
||||
self.written_bytes: int = 0
|
||||
|
||||
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
if not blob or self.stream_handle.closed:
|
||||
return False
|
||||
|
||||
def _decrypt_and_write():
|
||||
if self.stream_handle.closed:
|
||||
return False
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
self.stream_handle.seek(offset)
|
||||
_decrypted = blob.decrypt(
|
||||
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
|
||||
@@ -55,25 +55,33 @@ class StreamAssembler:
|
||||
self.stream_handle.write(_decrypted)
|
||||
self.stream_handle.flush()
|
||||
self.written_bytes += len(_decrypted)
|
||||
return True
|
||||
log.debug("decrypted %s", blob.blob_hash[:8])
|
||||
self.wrote_bytes_event.set()
|
||||
|
||||
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
if decrypted:
|
||||
log.info("decrypted %s", blob.blob_hash[:8])
|
||||
return
|
||||
await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
|
||||
async def setup(self):
|
||||
pass
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
pass
|
||||
|
||||
async def after_finished(self):
|
||||
pass
|
||||
|
||||
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
|
||||
if not os.path.isdir(output_dir):
|
||||
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
|
||||
await self.setup()
|
||||
self.sd_blob = await self.get_blob(self.sd_hash)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
|
||||
self.sd_blob)
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
self.output_path = await get_next_available_file_name(self.loop, output_dir,
|
||||
output_file_name or self.descriptor.suggested_file_name)
|
||||
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
await self.after_got_descriptor()
|
||||
self.stream_handle = open(self.output_path, 'wb')
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.sd_blob, self.descriptor
|
||||
@@ -85,17 +93,18 @@ class StreamAssembler:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
|
||||
break
|
||||
except ValueError as err:
|
||||
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash, str(err))
|
||||
except FileNotFoundError:
|
||||
log.debug("stream assembler stopped")
|
||||
return
|
||||
except (ValueError, IOError, OSError):
|
||||
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash)
|
||||
continue
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
|
||||
self.stream_finished_event.set()
|
||||
await self.after_finished()
|
||||
finally:
|
||||
self.stream_handle.close()
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
f = asyncio.Future(loop=self.loop)
|
||||
f.set_result(self.blob_manager.get_blob(blob_hash, length))
|
||||
return await f
|
||||
return self.blob_manager.get_blob(blob_hash, length)
|
||||
+61
-207
@@ -1,13 +1,14 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks, cancel_task
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
@@ -19,217 +20,70 @@ def drain_into(a: list, b: list):
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
|
||||
output_file_name: typing.Optional[str] = None,
|
||||
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
max_connections_per_stream: typing.Optional[int] = 8):
|
||||
class StreamDownloader(StreamAssembler):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_manager, sd_hash)
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.current_blob: 'BlobFile' = None
|
||||
|
||||
self.download_task: asyncio.Task = None
|
||||
self.accumulate_connections_task: asyncio.Task = None
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.output_dir = output_dir or os.getcwd()
|
||||
self.config = config
|
||||
self.output_dir = output_dir or self.config.download_dir
|
||||
self.output_file_name = output_file_name
|
||||
self._lock = asyncio.Lock(loop=self.loop)
|
||||
self.max_connections_per_stream = max_connections_per_stream
|
||||
self.fixed_peers = fixed_peers or []
|
||||
self.blob_downloader: typing.Optional[BlobDownloader] = None
|
||||
self.search_queue = asyncio.Queue(loop=loop)
|
||||
self.peer_queue = asyncio.Queue(loop=loop)
|
||||
self.accumulate_task: typing.Optional[asyncio.Task] = None
|
||||
self.descriptor: typing.Optional[StreamDescriptor]
|
||||
self.node: typing.Optional['Node'] = None
|
||||
self.assemble_task: typing.Optional[asyncio.Task] = None
|
||||
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
|
||||
|
||||
async def _update_current_blob(self, blob: 'BlobFile'):
|
||||
async with self._lock:
|
||||
drain_tasks(self.running_download_requests)
|
||||
self.current_blob = blob
|
||||
if not blob.get_is_verified():
|
||||
self._update_requests()
|
||||
async def setup(self): # start the peer accumulator and initialize the downloader
|
||||
if self.blob_downloader:
|
||||
raise Exception("downloader is already set up")
|
||||
if self.node:
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
|
||||
self.search_queue.put_nowait(self.sd_hash)
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.current_blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
|
||||
peer.address, peer.tcp_port, self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self._lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
async def after_got_descriptor(self):
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to search")
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.current_blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.current_blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.current_blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
async def after_finished(self):
|
||||
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
|
||||
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
|
||||
|
||||
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
|
||||
async with self._lock:
|
||||
if len(self.running_download_requests) < self.max_connections_per_stream:
|
||||
# update the running download requests
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
async with self._lock:
|
||||
if self.current_blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.current_blob
|
||||
else:
|
||||
for task in download_tasks:
|
||||
if task and not task.done():
|
||||
self.running_download_requests.append(task)
|
||||
return
|
||||
def stop(self):
|
||||
if self.accumulate_task:
|
||||
self.accumulate_task.cancel()
|
||||
self.accumulate_task = None
|
||||
if self.assemble_task:
|
||||
self.assemble_task.cancel()
|
||||
self.assemble_task = None
|
||||
if self.fixed_peers_handle:
|
||||
self.fixed_peers_handle.cancel()
|
||||
self.fixed_peers_handle = None
|
||||
self.blob_downloader = None
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
await self._update_current_blob(blob)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
return await self.blob_downloader.download_blob(blob_hash, length)
|
||||
|
||||
# the blob must be downloaded
|
||||
try:
|
||||
while not self.current_blob.get_is_verified():
|
||||
if not self.active_connections: # wait for a new connection
|
||||
await self.new_peer_event.wait()
|
||||
continue
|
||||
blob = await self.wait_for_download_or_new_peer()
|
||||
if blob:
|
||||
drain_tasks(self.running_download_requests)
|
||||
return blob
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
blob_queue = asyncio.Queue(loop=self.loop)
|
||||
blob_queue.put_nowait(self.sd_hash)
|
||||
task = asyncio.create_task(self.got_descriptor.wait())
|
||||
added_peers = asyncio.Event(loop=self.loop)
|
||||
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
|
||||
|
||||
if self.fixed_peers:
|
||||
def check_added_peers():
|
||||
if not added_peers.is_set():
|
||||
self._add_peer_protocols(self.fixed_peers)
|
||||
log.info("no dht peers for download yet, adding fixed peer")
|
||||
added_peers.set()
|
||||
|
||||
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
|
||||
|
||||
def got_descriptor(f):
|
||||
try:
|
||||
f.result()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
log.info("add head blob hash to peer search")
|
||||
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
|
||||
task.add_done_callback(got_descriptor)
|
||||
try:
|
||||
async with node.stream_peer_search_junction(blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
if not added_peers.is_set():
|
||||
added_peers.set()
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
log.info("cancelled head blob task")
|
||||
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
|
||||
add_fixed_peers_timer.cancel()
|
||||
|
||||
async def stop(self):
|
||||
cancel_task(self.accumulate_connections_task)
|
||||
self.accumulate_connections_task = None
|
||||
drain_tasks(self.running_download_requests)
|
||||
|
||||
while self.requested_from:
|
||||
_, peer_task_dict = self.requested_from.popitem()
|
||||
while peer_task_dict:
|
||||
peer, task = peer_task_dict.popitem()
|
||||
try:
|
||||
cancel_task(task)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
while self.active_connections:
|
||||
_, client = self.active_connections.popitem()
|
||||
if client:
|
||||
await client.close()
|
||||
log.info("stopped downloader")
|
||||
|
||||
async def _download(self):
|
||||
try:
|
||||
|
||||
log.info("download and decrypt stream")
|
||||
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
|
||||
log.info(
|
||||
"downloaded stream %s -> %s", self.sd_hash, self.output_path
|
||||
def add_fixed_peers(self):
|
||||
async def _add_fixed_peers():
|
||||
self.peer_queue.put_nowait([
|
||||
KademliaPeer(self.loop, address=(await resolve_host(url)), tcp_port=port + 1)
|
||||
for url, port in self.config.reflector_servers
|
||||
])
|
||||
if self.config.reflector_servers:
|
||||
self.fixed_peers_handle = self.loop.call_later(
|
||||
self.config.fixed_peer_delay if (
|
||||
'dht' not in self.config.components_to_skip
|
||||
and self.node
|
||||
and len(self.node.protocol.routing_table.get_peers())
|
||||
) else 0.0,
|
||||
lambda: self.loop.create_task(_add_fixed_peers())
|
||||
)
|
||||
await self.blob_manager.storage.change_file_status(
|
||||
self.descriptor.stream_hash, 'finished'
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
await self.stop()
|
||||
|
||||
def download(self, node: 'Node'):
|
||||
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.download_task = self.loop.create_task(self._download())
|
||||
def download(self, node: typing.Optional['Node'] = None):
|
||||
self.node = node
|
||||
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
|
||||
self.add_fixed_peers()
|
||||
@@ -2,13 +2,16 @@ import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.reflector.client import StreamReflectorClient
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.node import Node
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -20,7 +23,7 @@ class ManagedStream:
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
|
||||
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.download_directory = download_directory
|
||||
@@ -30,7 +33,6 @@ class ManagedStream:
|
||||
self.stream_hash = descriptor.stream_hash
|
||||
self.stream_claim_info = claim
|
||||
self._status = status
|
||||
self._store_after_finished: asyncio.Task = None
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
@@ -98,13 +100,17 @@ class ManagedStream:
|
||||
def sd_hash(self):
|
||||
return self.descriptor.sd_hash
|
||||
|
||||
@property
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = os.path.join(self.download_directory, self.file_name)
|
||||
if not os.path.exists(full_path):
|
||||
if not os.path.isfile(full_path):
|
||||
full_path = None
|
||||
mime_type = guess_media_type(os.path.basename(self.file_name))
|
||||
|
||||
if self.downloader:
|
||||
if self.downloader and self.downloader.written_bytes:
|
||||
written_bytes = self.downloader.written_bytes
|
||||
elif full_path:
|
||||
written_bytes = os.stat(full_path).st_size
|
||||
@@ -128,6 +134,7 @@ class ManagedStream:
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
'status': self.status,
|
||||
'claim_id': self.claim_id,
|
||||
'txid': self.txid,
|
||||
@@ -154,9 +161,13 @@ class ManagedStream:
|
||||
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
status=cls.STATUS_FINISHED)
|
||||
|
||||
async def stop_download(self):
|
||||
def start_download(self, node: typing.Optional['Node']):
|
||||
self.downloader.download(node)
|
||||
self.update_status(self.STATUS_RUNNING)
|
||||
|
||||
def stop_download(self):
|
||||
if self.downloader:
|
||||
await self.downloader.stop()
|
||||
self.downloader.stop()
|
||||
if not self.finished:
|
||||
self.update_status(self.STATUS_STOPPED)
|
||||
|
||||
@@ -177,6 +188,11 @@ class ManagedStream:
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd:
|
||||
sent.append(self.sd_hash)
|
||||
if not sent_sd and not needed:
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return []
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
@@ -193,4 +209,13 @@ class ManagedStream:
|
||||
protocol.transport.close()
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'ClaimDict'):
|
||||
self.stream_claim_info = StoredStreamClaim(
|
||||
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.serialized).decode(), claim.certificate_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
@@ -15,6 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.blob_manager = blob_manager
|
||||
self.descriptor = descriptor
|
||||
@@ -26,7 +27,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
log.info("Connected to reflector")
|
||||
log.debug("Connected to reflector")
|
||||
self.connected.set()
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
@@ -45,7 +46,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
return await self.pending_request
|
||||
finally:
|
||||
self.pending_request = None
|
||||
|
||||
@@ -37,7 +37,8 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
log.error("error receiving blob: %s", err)
|
||||
self.transport.close()
|
||||
return
|
||||
try:
|
||||
request = json.loads(data.decode())
|
||||
@@ -67,28 +68,33 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
self.send_response({"received_sd_blob": True})
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"received_sd_blob": True})
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
self.send_response({"received_sd_blob": False})
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.transport.close()
|
||||
self.send_response({"received_sd_blob": False})
|
||||
return
|
||||
else:
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
if 'blob_hash' not in request:
|
||||
|
||||
@@ -4,16 +4,20 @@ import typing
|
||||
import binascii
|
||||
import logging
|
||||
import random
|
||||
from lbrynet.error import ResolveError
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.extras.daemon.storage import lbc_to_dewies
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,7 +35,9 @@ filter_fields = [
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status'
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
comparison_operators = {
|
||||
@@ -45,42 +51,90 @@ comparison_operators = {
|
||||
|
||||
|
||||
class StreamManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
|
||||
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
|
||||
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node']):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.streams: typing.Set[ManagedStream] = set()
|
||||
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
|
||||
self.resume_downloading_task: asyncio.Task = None
|
||||
self.re_reflect_task: asyncio.Task = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.fixed_peers = fixed_peers
|
||||
self.reflector_servers = reflector_servers
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
stream.set_claim(claim_info, smart_decode(claim_info['value']))
|
||||
|
||||
async def start_stream(self, stream: ManagedStream) -> bool:
|
||||
"""
|
||||
Resume or rebuild a partial or completed stream
|
||||
"""
|
||||
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
|
||||
if not stream.running and not os.path.isfile(path):
|
||||
if stream.downloader:
|
||||
stream.downloader.stop()
|
||||
stream.downloader = None
|
||||
|
||||
# the directory is gone, can happen when the folder that contains a published file is deleted
|
||||
# reset the download directory to the default and update the file name
|
||||
if not os.path.isdir(stream.download_directory):
|
||||
stream.download_directory = self.config.download_dir
|
||||
|
||||
stream.downloader = self.make_downloader(
|
||||
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
|
||||
)
|
||||
if stream.status != ManagedStream.STATUS_FINISHED:
|
||||
await self.storage.change_file_status(stream.stream_hash, 'running')
|
||||
stream.update_status('running')
|
||||
stream.start_download(self.node)
|
||||
try:
|
||||
await asyncio.wait_for(self.loop.create_task(stream.downloader.got_descriptor.wait()),
|
||||
self.config.download_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
stream.stop_download()
|
||||
stream.downloader = None
|
||||
return False
|
||||
file_name = os.path.basename(stream.downloader.output_path)
|
||||
await self.storage.change_file_download_dir_and_file_name(
|
||||
stream.stream_hash, self.config.download_dir, file_name
|
||||
)
|
||||
self.wait_for_stream_finished(stream)
|
||||
return True
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
|
||||
return StreamDownloader(
|
||||
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
|
||||
)
|
||||
|
||||
async def add_stream(self, sd_hash: str, file_name: str, download_directory: str, status: str, claim):
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
if sd_blob.get_is_verified():
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, descriptor,
|
||||
download_directory,
|
||||
file_name,
|
||||
downloader, status, claim
|
||||
)
|
||||
self.streams.add(stream)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
async def load_streams_from_database(self):
|
||||
infos = await self.storage.get_all_lbry_files()
|
||||
for file_info in infos:
|
||||
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
|
||||
if sd_blob.get_is_verified():
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
downloader = StreamDownloader(
|
||||
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
|
||||
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, descriptor,
|
||||
binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(),
|
||||
downloader, file_info['status'], file_info['claim']
|
||||
)
|
||||
self.streams.add(stream)
|
||||
file_infos = await self.storage.get_all_lbry_files()
|
||||
await asyncio.gather(*[
|
||||
self.add_stream(
|
||||
file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
|
||||
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'], file_info['claim']
|
||||
) for file_info in file_infos
|
||||
])
|
||||
|
||||
async def resume(self):
|
||||
if not self.node:
|
||||
@@ -88,38 +142,45 @@ class StreamManager:
|
||||
return
|
||||
await self.node.joined.wait()
|
||||
resumed = 0
|
||||
for stream in self.streams:
|
||||
if stream.status == ManagedStream.STATUS_RUNNING:
|
||||
resumed += 1
|
||||
stream.downloader.download(self.node)
|
||||
self.wait_for_stream_finished(stream)
|
||||
t = [self.start_stream(stream) for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING]
|
||||
if resumed:
|
||||
log.info("resuming %i downloads", resumed)
|
||||
log.info("resuming %i downloads", t)
|
||||
await asyncio.gather(*t, loop=self.loop)
|
||||
|
||||
async def reflect_streams(self):
|
||||
streams = list(self.streams)
|
||||
batch = []
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= 10:
|
||||
await asyncio.gather(*batch)
|
||||
while True:
|
||||
if self.config.reflector_servers:
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
total = len(streams)
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
await asyncio.gather(*batch)
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
if total:
|
||||
log.info("uploaded %i streams to reflector", total)
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_streams_from_database()
|
||||
self.resume_downloading_task = self.loop.create_task(self.resume())
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
|
||||
async def stop(self):
|
||||
def stop(self):
|
||||
if self.resume_downloading_task and not self.resume_downloading_task.done():
|
||||
self.resume_downloading_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
stream = self.streams.pop()
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
|
||||
@@ -127,24 +188,21 @@ class StreamManager:
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams.add(stream)
|
||||
if self.reflector_servers:
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflector_servers:
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
self.streams.remove(stream)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
|
||||
blob_hashes = [stream.sd_hash]
|
||||
for blob_info in stream.descriptor.blobs[:-1]:
|
||||
blob_hashes.append(blob_info.blob_hash)
|
||||
for blob_hash in blob_hashes:
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
if blob.get_is_verified():
|
||||
await blob.delete()
|
||||
|
||||
await self.blob_manager.delete_blobs(blob_hashes)
|
||||
if delete_file:
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
if os.path.isfile(path):
|
||||
@@ -166,76 +224,70 @@ class StreamManager:
|
||||
)
|
||||
|
||||
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
|
||||
sd_blob_timeout: typing.Optional[float] = 60
|
||||
) -> typing.Optional[ManagedStream]:
|
||||
file_name: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
|
||||
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
|
||||
claim = smart_decode(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
|
||||
download_directory, file_name)
|
||||
try:
|
||||
downloader.download(node)
|
||||
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
|
||||
await downloader.got_descriptor.wait()
|
||||
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
log.info("stream timeout")
|
||||
await downloader.stop()
|
||||
downloader.stop()
|
||||
log.info("stopped stream")
|
||||
return
|
||||
file_name = os.path.basename(downloader.output_path)
|
||||
download_directory = os.path.dirname(downloader.output_path)
|
||||
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
|
||||
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
|
||||
data_rate
|
||||
downloader.descriptor.stream_hash, file_name, download_directory,
|
||||
0.0
|
||||
)
|
||||
await self.blob_manager.storage.save_content_claim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
|
||||
)
|
||||
|
||||
stored_claim = StoredStreamClaim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
|
||||
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
|
||||
claim_info.get('channel_name')
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
|
||||
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
|
||||
stored_claim)
|
||||
file_name, downloader, ManagedStream.STATUS_RUNNING)
|
||||
stream.set_claim(claim_info, claim)
|
||||
self.streams.add(stream)
|
||||
try:
|
||||
await stream.downloader.wrote_bytes_event.wait()
|
||||
self.wait_for_stream_finished(stream)
|
||||
return stream
|
||||
except asyncio.CancelledError:
|
||||
await downloader.stop()
|
||||
downloader.stop()
|
||||
log.debug("stopped stream")
|
||||
|
||||
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
async def download_stream_from_claim(self, node: 'Node', claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None,
|
||||
sd_blob_timeout: typing.Optional[float] = 60,
|
||||
timeout: typing.Optional[float] = 60,
|
||||
fee_amount: typing.Optional[float] = 0.0,
|
||||
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
fee_address: typing.Optional[str] = None,
|
||||
should_pay: typing.Optional[bool] = True) -> typing.Optional[ManagedStream]:
|
||||
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
if fee_address and fee_amount:
|
||||
if fee_amount > await self.wallet.default_account.get_balance():
|
||||
raise Exception("not enough funds")
|
||||
sd_hash = claim.source_hash.decode()
|
||||
if sd_hash in self.starting_streams:
|
||||
return await self.starting_streams[sd_hash]
|
||||
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
|
||||
if already_started:
|
||||
return already_started[0]
|
||||
if should_pay and fee_address and fee_amount and fee_amount > await self.wallet.default_account.get_balance():
|
||||
raise Exception("not enough funds")
|
||||
|
||||
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
|
||||
stream_task = self.loop.create_task(
|
||||
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
|
||||
self._download_stream_from_claim(node, self.config.download_dir, claim_info, file_name)
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(stream_task, sd_blob_timeout)
|
||||
await asyncio.wait_for(stream_task, timeout or self.config.download_timeout)
|
||||
stream = await stream_task
|
||||
self.starting_streams[sd_hash].set_result(stream)
|
||||
if fee_address and fee_amount:
|
||||
if should_pay and fee_address and fee_amount:
|
||||
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
|
||||
return stream
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
@@ -245,6 +297,11 @@ class StreamManager:
|
||||
del self.starting_streams[sd_hash]
|
||||
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
|
||||
comparison: typing.Optional[str] = None,
|
||||
**search_by) -> typing.List[ManagedStream]:
|
||||
@@ -280,3 +337,60 @@ class StreamManager:
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
file_name: typing.Optional[str] = None,
|
||||
timeout: typing.Optional[float] = None) -> typing.Optional[ManagedStream]:
|
||||
timeout = timeout or self.config.download_timeout
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise Exception("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
)
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
fee_address = claim.source_fee.address.decode()
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
|
||||
if not existing:
|
||||
existing.extend(self.get_filtered_streams(sd_hash=claim.source_hash.decode()))
|
||||
if existing and existing[0].claim_id != resolved['claim_id']:
|
||||
raise Exception(f"stream for {existing[0].claim_id} collides with existing "
|
||||
f"download {resolved['claim_id']}")
|
||||
elif not existing:
|
||||
existing.extend(self.get_filtered_streams(claim_id=resolved['claim_id']))
|
||||
if existing and existing[0].sd_hash != claim.source_hash.decode():
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
stream = await self.download_stream_from_claim(
|
||||
self.node, resolved, file_name, timeout, fee_amount, fee_address, False
|
||||
)
|
||||
log.info("started new stream, deleting old one")
|
||||
await self.delete_stream(existing[0])
|
||||
return stream
|
||||
elif existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
await self.start_stream(stream)
|
||||
return stream
|
||||
else:
|
||||
stream = existing[0]
|
||||
await self.start_stream(stream)
|
||||
return stream
|
||||
log.info("download stream from %s", uri)
|
||||
return await self.download_stream_from_claim(
|
||||
self.node, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
@@ -8,6 +8,7 @@ import json
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
import pkg_resources
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
@@ -136,3 +137,16 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
async def resolve_host(url: str) -> str:
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
except ValueError:
|
||||
pass
|
||||
loop = asyncio.get_running_loop()
|
||||
return (await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
))[0][4][0]
|
||||
@@ -0,0 +1,53 @@
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import socket
|
||||
import ipaddress
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("lbrynet")
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
async def main(blob_hash: str, url: str):
|
||||
conf = Config()
|
||||
loop = asyncio.get_running_loop()
|
||||
host_url, port = url.split(":")
|
||||
try:
|
||||
host = None
|
||||
if ipaddress.ip_address(host_url):
|
||||
host = host_url
|
||||
except ValueError:
|
||||
host = None
|
||||
if not host:
|
||||
host_info = await loop.getaddrinfo(
|
||||
host_url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
host = host_info[0][4][0]
|
||||
|
||||
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
|
||||
blob = blob_manager.get_blob(blob_hash)
|
||||
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
|
||||
conf.blob_download_timeout)
|
||||
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
|
||||
f"keep connection: {keep}")
|
||||
if blob.get_is_verified():
|
||||
await blob_manager.delete_blobs([blob.blob_hash])
|
||||
print(f"deleted {blob_hash}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
url = 'reflector.lbry.io:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
asyncio.run(main(sys.argv[1], url))
|
||||
@@ -0,0 +1,33 @@
|
||||
import os
|
||||
import binascii
|
||||
import sqlite3
|
||||
from lbrynet.conf import Config
|
||||
|
||||
|
||||
def main():
|
||||
conf = Config()
|
||||
db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite'))
|
||||
cur = db.cursor()
|
||||
files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall()
|
||||
update = {}
|
||||
for stream_hash, file_name, download_directory in files:
|
||||
try:
|
||||
binascii.unhexlify(file_name)
|
||||
except binascii.Error:
|
||||
try:
|
||||
binascii.unhexlify(download_directory)
|
||||
except binascii.Error:
|
||||
update[stream_hash] = (
|
||||
binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode()
|
||||
)
|
||||
if update:
|
||||
print(f"repair {len(update)} streams")
|
||||
for stream_hash, (file_name, download_directory) in update.items():
|
||||
cur.execute('update file set file_name=?, download_directory=? where stream_hash=?',
|
||||
(file_name, download_directory, stream_hash))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -57,31 +57,29 @@ def variance(times):
|
||||
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
|
||||
|
||||
|
||||
async def wait_for_done(api, uri):
|
||||
async def wait_for_done(conf, uri):
|
||||
name = uri.split("#")[0]
|
||||
last_complete = 0
|
||||
hang_count = 0
|
||||
while True:
|
||||
files = await api.file_list(claim_name=name)
|
||||
files = await daemon_rpc(conf, "file_list", claim_name=name)
|
||||
file = files[0]
|
||||
if file['status'] in ['finished', 'stopped']:
|
||||
return True, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
if last_complete < int(file['blobs_completed']):
|
||||
print(f"{file['blobs_completed']}/{file['blobs_in_stream']}...")
|
||||
hang_count = 0
|
||||
last_complete = int(file['blobs_completed'])
|
||||
else:
|
||||
hang_count += 1
|
||||
await asyncio.sleep(1.0)
|
||||
if hang_count > 30:
|
||||
if hang_count > 10:
|
||||
return False, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
|
||||
|
||||
async def main(start_daemon=True, uris=None):
|
||||
async def main(uris=None):
|
||||
if not uris:
|
||||
uris = await get_frontpage_uris()
|
||||
conf = Config()
|
||||
daemon = None
|
||||
try:
|
||||
await daemon_rpc(conf, 'status')
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
@@ -113,12 +111,12 @@ async def main(start_daemon=True, uris=None):
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
|
||||
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
|
||||
# downloaded, msg, blobs_in_stream = await wait_for_done(conf, uri)
|
||||
# if downloaded:
|
||||
# downloaded_times.append((time.time()-start) / downloaded)
|
||||
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
|
||||
# print(f"\tdownloaded {uri} @ {(time.time()-start) / blobs_in_stream} seconds per blob")
|
||||
# else:
|
||||
# print(f"failed to downlload {uri}, got {msg}")
|
||||
# print(f"\tfailed to download {uri}, got {msg}")
|
||||
# download_failures.append(uri)
|
||||
except:
|
||||
print(f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}")
|
||||
@@ -136,8 +134,7 @@ async def main(start_daemon=True, uris=None):
|
||||
nt = '\n\t'
|
||||
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
|
||||
print(result)
|
||||
if daemon:
|
||||
await daemon.shutdown()
|
||||
|
||||
# webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
|
||||
# if webhook:
|
||||
# await report_to_slack(result, webhook)
|
||||
|
||||
@@ -41,9 +41,10 @@ class CommandTestCase(IntegrationTestCase):
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_uploads = False
|
||||
conf.reflect_streams = False
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('localhost', 50001)]
|
||||
conf.reflector_servers = []
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
|
||||
import shutil
|
||||
import os
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobRequest
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
@@ -14,6 +18,12 @@ from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
# logging.getLogger("lbrynet").setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def mock_config():
|
||||
config = Config()
|
||||
config.fixed_peer_delay = 10000
|
||||
return config
|
||||
|
||||
|
||||
class BlobExchangeTestBase(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
@@ -22,12 +32,15 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.server_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.client_dir)
|
||||
self.addCleanup(shutil.rmtree, self.server_dir)
|
||||
|
||||
self.server_storage = SQLiteStorage(Config(), os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
|
||||
self.client_storage = SQLiteStorage(Config(), os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
|
||||
reflector_servers=[])
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
@@ -53,12 +66,10 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
async def _test_transfer_blob(self, blob_hash: str):
|
||||
client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
downloaded = await request_blob(self.loop, client_blob, protocol, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2)
|
||||
await protocol.close()
|
||||
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2, 3)
|
||||
self.assertEqual(client_blob.get_is_verified(), True)
|
||||
self.assertTrue(downloaded)
|
||||
|
||||
@@ -91,17 +102,15 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(blob_hash)
|
||||
)
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_host_different_blobs_to_multiple_peers_at_once(self):
|
||||
@@ -125,14 +134,25 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(sd_hash)
|
||||
)
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_server_chunked_request(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address)
|
||||
transport = asyncio.Transport(extra={'peername': ('ip', 90)})
|
||||
received_data = BytesIO()
|
||||
transport.write = received_data.write
|
||||
server_protocol.connection_made(transport)
|
||||
blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize()
|
||||
for byte in blob_request:
|
||||
server_protocol.data_received(bytes([byte]))
|
||||
await asyncio.sleep(0.1) # yield execution
|
||||
self.assertTrue(len(received_data.getvalue()) > 0)
|
||||
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbrynet.dht.serialization.bencoding import bencode, bdecode
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
@@ -67,6 +69,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
|
||||
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
|
||||
)
|
||||
peer2_from_peer1.update_tcp_port(3333)
|
||||
peer3 = peer1.peer_manager.get_kademlia_peer(
|
||||
constants.generate_id(), '1.2.3.6', udp_port=4444
|
||||
)
|
||||
@@ -76,14 +79,14 @@ class TestProtocol(AsyncioTestCase):
|
||||
self.assertEqual(True, peer1.data_store.has_peers_for_blob(b'2' * 48))
|
||||
self.assertEqual(False, peer1.data_store.has_peers_for_blob(b'3' * 48))
|
||||
self.assertListEqual([peer2_from_peer1], peer1.data_store.get_storing_contacts())
|
||||
|
||||
peer1.data_store.completed_blobs.add(binascii.hexlify(b'2' * 48).decode())
|
||||
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
|
||||
self.assertSetEqual(
|
||||
{b'2' * 48, b'token', b'protocolVersion'}, set(find_value_response.keys())
|
||||
)
|
||||
self.assertEqual(1, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1)
|
||||
# self.assertEqual(peer2_from_peer1.tcp_port, 3333)
|
||||
self.assertEqual(2, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
|
||||
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
|
||||
|
||||
peer1.stop()
|
||||
peer2.stop()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import unittest
|
||||
import asyncio
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -41,16 +42,17 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
|
||||
async def test_yield_order(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_one_stopped_first(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.101, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
@@ -62,16 +64,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
for i in range(10):
|
||||
if i == 5:
|
||||
return
|
||||
await asyncio.sleep(0.101)
|
||||
await asyncio.sleep(0.2)
|
||||
yield 1
|
||||
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen(), slow_gen)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
async def test_stop_when_encapsulating_task_cancelled(self):
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
|
||||
async def _task():
|
||||
async with AsyncGeneratorJunction(self.loop) as junction:
|
||||
@@ -81,7 +83,7 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
pass
|
||||
|
||||
task = self.loop.create_task(_task())
|
||||
self.loop.call_later(0.5, task.cancel)
|
||||
self.loop.call_later(1.0, task.cancel)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
|
||||
@@ -84,15 +84,6 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_fee_and_ungenerous_data(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
fake_fee_amount = 4.5
|
||||
correct_result = size / 10 ** 6 * conf.data_rate + fake_fee_amount
|
||||
daemon = get_test_daemon(conf, with_fee=True)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
def test_generous_data_and_no_fee(self):
|
||||
size = 10000000
|
||||
correct_result = 0.0
|
||||
@@ -100,14 +91,6 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_ungenerous_data_and_no_fee(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
correct_result = size / 10 ** 6 * conf.data_rate
|
||||
daemon = get_test_daemon(conf)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
|
||||
@unittest.SkipTest
|
||||
class TestJsonRpc(unittest.TestCase):
|
||||
|
||||
@@ -7,8 +7,8 @@ from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
|
||||
class TestStreamAssembler(AsyncioTestCase):
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
import contextlib
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.dht.node import Node
|
||||
@@ -21,59 +22,58 @@ class TestStreamDownloader(BlobExchangeTestBase):
|
||||
f.write(self.stream_bytes)
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
|
||||
self.sd_hash = descriptor.calculate_sd_hash()
|
||||
self.downloader = StreamDownloader(self.loop, self.client_blob_manager, self.sd_hash, 3, 3, self.client_dir)
|
||||
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_peer_search=None):
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
|
||||
await self.setup_stream(blob_count)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _mock_peer_search(*_):
|
||||
async def _gen():
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
q2.put_nowait([self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
yield _gen()
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search or _mock_peer_search
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await asyncio.sleep(0.01)
|
||||
self.assertTrue(self.downloader.stream_handle.closed)
|
||||
|
||||
async def test_transfer_stream(self):
|
||||
await self._test_transfer_stream(10)
|
||||
|
||||
# async def test_transfer_hundred_blob_stream(self):
|
||||
# await self._test_transfer_stream(100)
|
||||
@unittest.SkipTest
|
||||
async def test_transfer_hundred_blob_stream(self):
|
||||
await self._test_transfer_stream(100)
|
||||
|
||||
async def test_transfer_stream_bad_first_peer_good_second(self):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def mock_peer_search(*_):
|
||||
async def _gen():
|
||||
await asyncio.sleep(0.05, loop=self.loop)
|
||||
yield [bad_peer]
|
||||
await asyncio.sleep(0.1, loop=self.loop)
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
|
||||
yield _gen()
|
||||
q2.put_nowait([bad_peer])
|
||||
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
|
||||
@@ -21,7 +21,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
self.stream_manager = StreamManager(self.loop, self.blob_manager, None, self.storage, None, 3.0, 3.0)
|
||||
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
|
||||
|
||||
Reference in new issue
Block a user