Compare commits

..
1 Commits
Author SHA1 Message Date
Lex Berezhny e01fbc6f39 initial import of android lbrysdk build 2019-01-27 22:45:32 -05:00
68 changed files with 3560 additions and 1138 deletions

No files matched your search

+3 -1
View File
@@ -33,7 +33,9 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
# extension-pkg-whitelist=
extension-pkg-whitelist=
miniupnpc,
unqlite
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
+3 -3
View File
@@ -1,12 +1,12 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
+5
View File
@@ -0,0 +1,5 @@
build:
python setup.py apk
adb install -r lbry-0.1-service-debug.apk
.PHONY: build
+49
View File
@@ -0,0 +1,49 @@
import inspect
from os.path import dirname, join
import sh
from pythonforandroid.util import ensure_dir
from pythonforandroid.toolchain import (
Bootstrap, current_directory, info, info_main, shprint
)
class LBRYServiceBootstrap(Bootstrap):
name = 'lbry-service'
recipe_depends = ['genericndkbuild', 'python3']
bootstrap_dir = dirname(__file__)
def get_common_dir(self):
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
def run_distribute(self):
info_main('# Creating Android project from build and {} bootstrap'.format(
self.name))
info('This currently just copies the build stuff straight from the build dir.')
shprint(sh.rm, '-rf', self.dist_dir)
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Copying python distribution')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
python_bundle_dir = join('_python_bundle', '_python_bundle')
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super().run_distribute()
+89
View File
@@ -0,0 +1,89 @@
# prevent user to include invalid extensions
*.apk
*.pxd
# eggs
*.egg-info
# unit test
unittest/*
# python config
config/makesetup
# unused kivy files (platform specific)
kivy/input/providers/wm_*
kivy/input/providers/mactouch*
kivy/input/providers/probesysfs*
kivy/input/providers/mtdev*
kivy/input/providers/hidinput*
kivy/core/camera/camera_videocapture*
kivy/core/spelling/*osx*
kivy/core/video/video_pyglet*
kivy/tools
kivy/tests/*
kivy/*/*.h
kivy/*/*.pxi
# unused encodings
lib-dynload/*codec*
encodings/cp*.pyo
encodings/tis*
encodings/shift*
encodings/bz2*
encodings/iso*
encodings/undefined*
encodings/johab*
encodings/p*
encodings/m*
encodings/euc*
encodings/k*
encodings/unicode_internal*
encodings/quo*
encodings/gb*
encodings/big5*
encodings/hp*
encodings/hz*
# unused python modules
bsddb/*
wsgiref/*
hotshot/*
pydoc_data/*
tty.pyo
anydbm.pyo
nturl2path.pyo
LICENCE.txt
macurl2path.pyo
dummy_threading.pyo
audiodev.pyo
antigravity.pyo
dumbdbm.pyo
sndhdr.pyo
__phello__.foo.pyo
sunaudio.pyo
os2emxpath.pyo
multiprocessing/dummy*
# unused binaries python modules
lib-dynload/termios.so
lib-dynload/_lsprof.so
lib-dynload/*audioop.so
lib-dynload/_hotshot.so
lib-dynload/_heapq.so
lib-dynload/_json.so
lib-dynload/grp.so
lib-dynload/resource.so
lib-dynload/pyexpat.so
lib-dynload/_ctypes_test.so
lib-dynload/_testcapi.so
# odd files
plat-linux3/regen
#>sqlite3
# conditionnal include depending if some recipes are included or not.
sqlite3/*
lib-dynload/_sqlite3.so
#<sqlite3
+1
View File
@@ -0,0 +1 @@
include $(call all-subdir-makefiles)
@@ -0,0 +1,7 @@
# Uncomment this if you're using STL in your project
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
# APP_STL := stlport_static
# APP_ABI := armeabi armeabi-v7a x86
APP_ABI := $(ARCH)
@@ -0,0 +1,22 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
# Add your application source files here...
LOCAL_SRC_FILES := start.c pyjniusjni.c
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
LOCAL_SHARED_LIBRARIES := python_shared
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
include $(BUILD_SHARED_LIBRARY)
ifdef CRYSTAX_PYTHON_VERSION
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
endif
@@ -0,0 +1,10 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
LOCAL_SRC_FILES := YourSourceHere.c
include $(BUILD_SHARED_LIBRARY)
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
@@ -0,0 +1,6 @@
#define BOOTSTRAP_NAME_SERVICEONLY
#define BOOTSTRAP_USES_NO_SDL_HEADERS
const char bootstrap_name[] = "service_only";
@@ -0,0 +1,103 @@
#include <pthread.h>
#include <jni.h>
#define LOGI(...) do {} while (0)
#define LOGE(...) do {} while (0)
#include "android/log.h"
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
/* #define LOGP(x) LOG("python", (x)) */
#define LOG_TAG "Python_android"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
/* Function headers */
JNIEnv* Android_JNI_GetEnv(void);
static void Android_JNI_ThreadDestroyed(void*);
static pthread_key_t mThreadKey;
static JavaVM* mJavaVM;
int Android_JNI_SetupThread(void)
{
Android_JNI_GetEnv();
return 1;
}
/* Library init */
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
{
JNIEnv *env;
mJavaVM = vm;
LOGI("JNI_OnLoad called");
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
LOGE("Failed to get the environment using GetEnv()");
return -1;
}
/*
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
*/
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
}
Android_JNI_SetupThread();
return JNI_VERSION_1_4;
}
JNIEnv* Android_JNI_GetEnv(void)
{
/* From http://developer.android.com/guide/practices/jni.html
* All threads are Linux threads, scheduled by the kernel.
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
* and cannot make JNI calls.
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
* is a no-op.
* Note: You can call this function any number of times for the same thread, there's no harm in it
*/
JNIEnv *env;
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
if(status < 0) {
LOGE("failed to attach current thread");
return 0;
}
/* From http://developer.android.com/guide/practices/jni.html
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
* Note: The destructor is not called unless the stored value is != NULL
* Note: You can call this function any number of times for the same thread, there's no harm in it
* (except for some lost CPU cycles)
*/
pthread_setspecific(mThreadKey, (void*) env);
return env;
}
static void Android_JNI_ThreadDestroyed(void* value)
{
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
JNIEnv *env = (JNIEnv*) value;
if (env != NULL) {
(*mJavaVM)->DetachCurrentThread(mJavaVM);
pthread_setspecific(mThreadKey, NULL);
}
}
void *WebView_AndroidGetJNIEnv()
{
return Android_JNI_GetEnv();
}
Whitespace-only changes.
@@ -0,0 +1,141 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class Octal {
/**
* Parse an octal string from a header buffer. This is used for the file
* permission mode value.
*
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal string.
*/
public static long parseOctal(byte[] header, int offset, int length) {
long result = 0;
boolean stillPadding = true;
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
if (header[i] == (byte) ' ' || header[i] == '0') {
if (stillPadding)
continue;
if (header[i] == (byte) ' ')
break;
}
stillPadding = false;
result = ( result << 3 ) + ( header[i] - '0' );
}
return result;
}
/**
* Parse an octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The integer value of the octal bytes.
*/
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
int idx = length - 1;
buf[offset + idx] = 0;
--idx;
buf[offset + idx] = (byte) ' ';
--idx;
if (value == 0) {
buf[offset + idx] = (byte) '0';
--idx;
} else {
for (long val = value; idx >= 0 && val > 0; --idx) {
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
val = val >> 3;
}
}
for (; idx >= 0; --idx) {
buf[offset + idx] = (byte) ' ';
}
return offset + length;
}
/**
* Parse the checksum octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The integer value of the entry's checksum.
*/
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
getOctalBytes( value, buf, offset, length );
buf[offset + length - 1] = (byte) ' ';
buf[offset + length - 2] = 0;
return offset + length;
}
/**
* Parse an octal long integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal bytes.
*/
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
byte[] temp = new byte[length + 1];
getOctalBytes( value, temp, 0, length + 1 );
System.arraycopy( temp, 0, buf, offset, length );
return offset + length;
}
}
@@ -0,0 +1,28 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class TarConstants {
public static final int EOF_BLOCK = 1024;
public static final int DATA_BLOCK = 512;
public static final int HEADER_BLOCK = 512;
}
@@ -0,0 +1,284 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
import java.util.Date;
/**
* @author Kamran Zafar
*
*/
public class TarEntry {
protected File file;
protected TarHeader header;
private TarEntry() {
this.file = null;
header = new TarHeader();
}
public TarEntry(File file, String entryName) {
this();
this.file = file;
this.extractTarHeader(entryName);
}
public TarEntry(byte[] headerBuf) {
this();
this.parseTarHeader(headerBuf);
}
/**
* Constructor to create an entry from an existing TarHeader object.
*
* This method is useful to add new entries programmatically (e.g. for
* adding files or directories that do not exist in the file system).
*
* @param header
*
*/
public TarEntry(TarHeader header) {
this.file = null;
this.header = header;
}
public boolean equals(TarEntry it) {
return header.name.toString().equals(it.header.name.toString());
}
public boolean isDescendent(TarEntry desc) {
return desc.header.name.toString().startsWith(header.name.toString());
}
public TarHeader getHeader() {
return header;
}
public String getName() {
String name = header.name.toString();
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
name = header.namePrefix.toString() + "/" + name;
}
return name;
}
public void setName(String name) {
header.name = new StringBuffer(name);
}
public int getUserId() {
return header.userId;
}
public void setUserId(int userId) {
header.userId = userId;
}
public int getGroupId() {
return header.groupId;
}
public void setGroupId(int groupId) {
header.groupId = groupId;
}
public String getUserName() {
return header.userName.toString();
}
public void setUserName(String userName) {
header.userName = new StringBuffer(userName);
}
public String getGroupName() {
return header.groupName.toString();
}
public void setGroupName(String groupName) {
header.groupName = new StringBuffer(groupName);
}
public void setIds(int userId, int groupId) {
this.setUserId(userId);
this.setGroupId(groupId);
}
public void setModTime(long time) {
header.modTime = time / 1000;
}
public void setModTime(Date time) {
header.modTime = time.getTime() / 1000;
}
public Date getModTime() {
return new Date(header.modTime * 1000);
}
public File getFile() {
return this.file;
}
public long getSize() {
return header.size;
}
public void setSize(long size) {
header.size = size;
}
/**
* Checks if the org.kamrazafar.jtar entry is a directory
*
* @return
*/
public boolean isDirectory() {
if (this.file != null)
return this.file.isDirectory();
if (header != null) {
if (header.linkFlag == TarHeader.LF_DIR)
return true;
if (header.name.toString().endsWith("/"))
return true;
}
return false;
}
/**
* Extract header from File
*
* @param entryName
*/
public void extractTarHeader(String entryName) {
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
}
/**
* Calculate checksum
*
* @param buf
* @return
*/
public long computeCheckSum(byte[] buf) {
long sum = 0;
for (int i = 0; i < buf.length; ++i) {
sum += 255 & buf[i];
}
return sum;
}
/**
* Writes the header to the byte buffer
*
* @param outbuf
*/
public void writeEntryHeader(byte[] outbuf) {
int offset = 0;
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
long size = header.size;
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
int csOffset = offset;
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
outbuf[offset++] = (byte) ' ';
outbuf[offset++] = header.linkFlag;
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
for (; offset < outbuf.length;)
outbuf[offset++] = 0;
long checkSum = this.computeCheckSum(outbuf);
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
}
/**
* Parses the tar header to the byte buffer
*
* @param header
* @param bh
*/
public void parseTarHeader(byte[] bh) {
int offset = 0;
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
offset += TarHeader.MODELEN;
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
offset += TarHeader.UIDLEN;
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
offset += TarHeader.GIDLEN;
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
offset += TarHeader.SIZELEN;
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
offset += TarHeader.MODTIMELEN;
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
offset += TarHeader.CHKSUMLEN;
header.linkFlag = bh[offset++];
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
offset += TarHeader.USTAR_MAGICLEN;
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
offset += TarHeader.USTAR_USER_NAMELEN;
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset += TarHeader.USTAR_GROUP_NAMELEN;
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
}
}
@@ -0,0 +1,243 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* Header
*
* <pre>
* Offset Size Field
* 0 100 File name
* 100 8 File mode
* 108 8 Owner's numeric user ID
* 116 8 Group's numeric user ID
* 124 12 File size in bytes
* 136 12 Last modification time in numeric Unix time format
* 148 8 Checksum for header block
* 156 1 Link indicator (file type)
* 157 100 Name of linked file
* </pre>
*
*
* File Types
*
* <pre>
* Value Meaning
* '0' Normal file
* (ASCII NUL) Normal file (now obsolete)
* '1' Hard link
* '2' Symbolic link
* '3' Character special
* '4' Block special
* '5' Directory
* '6' FIFO
* '7' Contigous
* </pre>
*
*
*
* Ustar header
*
* <pre>
* Offset Size Field
* 257 6 UStar indicator "ustar"
* 263 2 UStar version "00"
* 265 32 Owner user name
* 297 32 Owner group name
* 329 8 Device major number
* 337 8 Device minor number
* 345 155 Filename prefix
* </pre>
*/
public class TarHeader {
/*
* Header
*/
public static final int NAMELEN = 100;
public static final int MODELEN = 8;
public static final int UIDLEN = 8;
public static final int GIDLEN = 8;
public static final int SIZELEN = 12;
public static final int MODTIMELEN = 12;
public static final int CHKSUMLEN = 8;
public static final byte LF_OLDNORM = 0;
/*
* File Types
*/
public static final byte LF_NORMAL = (byte) '0';
public static final byte LF_LINK = (byte) '1';
public static final byte LF_SYMLINK = (byte) '2';
public static final byte LF_CHR = (byte) '3';
public static final byte LF_BLK = (byte) '4';
public static final byte LF_DIR = (byte) '5';
public static final byte LF_FIFO = (byte) '6';
public static final byte LF_CONTIG = (byte) '7';
/*
* Ustar header
*/
public static final String USTAR_MAGIC = "ustar"; // POSIX
public static final int USTAR_MAGICLEN = 8;
public static final int USTAR_USER_NAMELEN = 32;
public static final int USTAR_GROUP_NAMELEN = 32;
public static final int USTAR_DEVLEN = 8;
public static final int USTAR_FILENAME_PREFIX = 155;
// Header values
public StringBuffer name;
public int mode;
public int userId;
public int groupId;
public long size;
public long modTime;
public int checkSum;
public byte linkFlag;
public StringBuffer linkName;
public StringBuffer magic; // ustar indicator and version
public StringBuffer userName;
public StringBuffer groupName;
public int devMajor;
public int devMinor;
public StringBuffer namePrefix;
public TarHeader() {
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
this.name = new StringBuffer();
this.linkName = new StringBuffer();
String user = System.getProperty("user.name", "");
if (user.length() > 31)
user = user.substring(0, 31);
this.userId = 0;
this.groupId = 0;
this.userName = new StringBuffer(user);
this.groupName = new StringBuffer("");
this.namePrefix = new StringBuffer();
}
/**
* Parse an entry name from a header buffer.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The header's entry name.
*/
public static StringBuffer parseName(byte[] header, int offset, int length) {
StringBuffer result = new StringBuffer(length);
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
result.append((char) header[i]);
}
return result;
}
/**
* Determine the number of bytes in an entry name.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The number of bytes in a header's entry name.
*/
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
int i;
for (i = 0; i < length && i < name.length(); ++i) {
buf[offset + i] = (byte) name.charAt(i);
}
for (; i < length; ++i) {
buf[offset + i] = 0;
}
return offset + length;
}
/**
* Creates a new header for a file/directory entry.
*
*
* @param name
* File name
* @param size
* File size in bytes
* @param modTime
* Last modification time in numeric Unix time format
* @param dir
* Is directory
*
* @return
*/
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
String name = entryName;
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
TarHeader header = new TarHeader();
header.linkName = new StringBuffer("");
if (name.length() > 100) {
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
} else {
header.name = new StringBuffer(name);
}
if (dir) {
header.mode = 040755;
header.linkFlag = TarHeader.LF_DIR;
if (header.name.charAt(header.name.length() - 1) != '/') {
header.name.append("/");
}
header.size = 0;
} else {
header.mode = 0100644;
header.linkFlag = TarHeader.LF_NORMAL;
header.size = size;
}
header.modTime = modTime;
header.checkSum = 0;
header.devMajor = 0;
header.devMinor = 0;
return header;
}
}
@@ -0,0 +1,249 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* @author Kamran Zafar
*
*/
public class TarInputStream extends FilterInputStream {
private static final int SKIP_BUFFER_SIZE = 2048;
private TarEntry currentEntry;
private long currentFileSize;
private long bytesRead;
private boolean defaultSkip = false;
public TarInputStream(InputStream in) {
super(in);
currentFileSize = 0;
bytesRead = 0;
}
@Override
public boolean markSupported() {
return false;
}
/**
* Not supported
*
*/
@Override
public synchronized void mark(int readlimit) {
}
/**
* Not supported
*
*/
@Override
public synchronized void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
/**
* Read a byte
*
* @see java.io.FilterInputStream#read()
*/
@Override
public int read() throws IOException {
byte[] buf = new byte[1];
int res = this.read(buf, 0, 1);
if (res != -1) {
return 0xFF & buf[0];
}
return res;
}
/**
* Checks if the bytes being read exceed the entry size and adjusts the byte
* array length. Updates the byte counters
*
*
* @see java.io.FilterInputStream#read(byte[], int, int)
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (currentEntry != null) {
if (currentFileSize == currentEntry.getSize()) {
return -1;
} else if ((currentEntry.getSize() - currentFileSize) < len) {
len = (int) (currentEntry.getSize() - currentFileSize);
}
}
int br = super.read(b, off, len);
if (br != -1) {
if (currentEntry != null) {
currentFileSize += br;
}
bytesRead += br;
}
return br;
}
/**
* Returns the next entry in the tar file
*
* @return TarEntry
* @throws IOException
*/
public TarEntry getNextEntry() throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
int tr = 0;
// Read full header
while (tr < TarConstants.HEADER_BLOCK) {
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
if (res < 0) {
break;
}
System.arraycopy(theader, 0, header, tr, res);
tr += res;
}
// Check if record is null
boolean eof = true;
for (byte b : header) {
if (b != 0) {
eof = false;
break;
}
}
if (!eof) {
currentEntry = new TarEntry(header);
}
return currentEntry;
}
/**
* Returns the current offset (in bytes) from the beginning of the stream.
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
*/
public long getCurrentOffset() {
return bytesRead;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
// Not fully read, skip rest of the bytes
long bs = 0;
while (bs < currentEntry.getSize() - currentFileSize) {
long res = skip(currentEntry.getSize() - currentFileSize - bs);
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
// I suspect file corruption
throw new IOException("Possible tar file corruption");
}
bs += res;
}
}
currentEntry = null;
currentFileSize = 0L;
skipPad();
}
}
/**
* Skips the pad at the end of each tar entry file content
*
* @throws IOException
*/
protected void skipPad() throws IOException {
if (bytesRead > 0) {
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
if (extra > 0) {
long bs = 0;
while (bs < TarConstants.DATA_BLOCK - extra) {
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
bs += res;
}
}
}
}
/**
* Skips 'n' bytes on the InputStream<br>
* Overrides default implementation of skip
*
*/
@Override
public long skip(long n) throws IOException {
if (defaultSkip) {
// use skip method of parent stream
// may not work if skip not implemented by parent
long bs = super.skip(n);
bytesRead += bs;
return bs;
}
if (n <= 0) {
return 0;
}
long left = n;
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
while (left > 0) {
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
if (res < 0) {
break;
}
left -= res;
}
return n - left;
}
public boolean isDefaultSkip() {
return defaultSkip;
}
public void setDefaultSkip(boolean defaultSkip) {
this.defaultSkip = defaultSkip;
}
}
@@ -0,0 +1,163 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
/**
* @author Kamran Zafar
*
*/
public class TarOutputStream extends OutputStream {
private final OutputStream out;
private long bytesWritten;
private long currentFileSize;
private TarEntry currentEntry;
public TarOutputStream(OutputStream out) {
this.out = out;
bytesWritten = 0;
currentFileSize = 0;
}
public TarOutputStream(final File fout) throws FileNotFoundException {
this.out = new BufferedOutputStream(new FileOutputStream(fout));
bytesWritten = 0;
currentFileSize = 0;
}
/**
* Opens a file for writing.
*/
public TarOutputStream(final File fout, final boolean append) throws IOException {
@SuppressWarnings("resource")
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
final long fileSize = fout.length();
if (append && fileSize > TarConstants.EOF_BLOCK) {
raf.seek(fileSize - TarConstants.EOF_BLOCK);
}
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
}
/**
* Appends the EOF record and closes the stream
*
* @see java.io.FilterOutputStream#close()
*/
@Override
public void close() throws IOException {
closeCurrentEntry();
write( new byte[TarConstants.EOF_BLOCK] );
out.close();
}
/**
* Writes a byte to the stream and updates byte counters
*
* @see java.io.FilterOutputStream#write(int)
*/
@Override
public void write(int b) throws IOException {
out.write( b );
bytesWritten += 1;
if (currentEntry != null) {
currentFileSize += 1;
}
}
/**
* Checks if the bytes being written exceed the current entry size.
*
* @see java.io.FilterOutputStream#write(byte[], int, int)
*/
@Override
public void write(byte[] b, int off, int len) throws IOException {
if (currentEntry != null && !currentEntry.isDirectory()) {
if (currentEntry.getSize() < currentFileSize + len) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
+ "] being written." );
}
}
out.write( b, off, len );
bytesWritten += len;
if (currentEntry != null) {
currentFileSize += len;
}
}
/**
* Writes the next tar entry header on the stream
*
* @param entry
* @throws IOException
*/
public void putNextEntry(TarEntry entry) throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
entry.writeEntryHeader( header );
write( header );
currentEntry = entry;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
+ currentEntry.getSize() + "] has not been fully written." );
}
currentEntry = null;
currentFileSize = 0;
pad();
}
}
/**
* Pads the last content block
*
* @throws IOException
*/
protected void pad() throws IOException {
if (bytesWritten > 0) {
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
if (extra > 0) {
write( new byte[TarConstants.DATA_BLOCK - extra] );
}
}
}
}
@@ -0,0 +1,96 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* @author Kamran
*
*/
public class TarUtils {
/**
* Determines the tar file size of the given folder/file path
*
* @param path
* @return
*/
public static long calculateTarSize(File path) {
return tarSize(path) + TarConstants.EOF_BLOCK;
}
private static long tarSize(File dir) {
long size = 0;
if (dir.isFile()) {
return entrySize(dir.length());
} else {
File[] subFiles = dir.listFiles();
if (subFiles != null && subFiles.length > 0) {
for (File file : subFiles) {
if (file.isFile()) {
size += entrySize(file.length());
} else {
size += tarSize(file);
}
}
} else {
// Empty folder header
return TarConstants.HEADER_BLOCK;
}
}
return size;
}
private static long entrySize(long fileSize) {
long size = 0;
size += TarConstants.HEADER_BLOCK; // Header
size += fileSize; // File size
long extra = size % TarConstants.DATA_BLOCK;
if (extra > 0) {
size += (TarConstants.DATA_BLOCK - extra); // pad
}
return size;
}
public static String trim(String s, char c) {
StringBuffer tmp = new StringBuffer(s);
for (int i = 0; i < tmp.length(); i++) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
for (int i = tmp.length() - 1; i >= 0; i--) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
return tmp.toString();
}
}
@@ -0,0 +1,400 @@
package org.kivy.android;
import java.net.Socket;
import java.net.InetSocketAddress;
import android.os.SystemClock;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
import android.app.*;
import android.content.*;
import android.view.*;
import android.view.SurfaceView;
import android.app.Activity;
import android.content.Intent;
import android.util.Log;
import android.widget.Toast;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.PowerManager;
import android.graphics.PixelFormat;
import android.view.SurfaceHolder;
import android.content.Context;
import android.content.pm.ActivityInfo;
import android.content.pm.PackageManager;
import android.content.pm.ApplicationInfo;
import android.content.Intent;
import android.widget.ImageView;
import java.io.InputStream;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Color;
import android.widget.AbsoluteLayout;
import android.webkit.WebViewClient;
import android.webkit.WebView;
import org.kivy.android.PythonUtil;
import org.renpy.android.ResourceManager;
import org.renpy.android.AssetExtract;
public class PythonActivity extends Activity {
// This activity is modified from a mixture of the SDLActivity and
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
// specifics.
private static final String TAG = "PythonActivity";
public static PythonActivity mActivity = null;
/** If shared libraries (e.g. the native application) could not be loaded. */
public static boolean mBrokenLibraries;
protected static Thread mPythonThread;
private ResourceManager resourceManager = null;
private Bundle mMetaData = null;
private PowerManager.WakeLock mWakeLock = null;
public String getAppRoot() {
String app_root = getFilesDir().getAbsolutePath() + "/app";
return app_root;
}
public static void initialize() {
// The static nature of the singleton and Android quirkiness force us to initialize everything here
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
mBrokenLibraries = false;
}
@Override
protected void onCreate(Bundle savedInstanceState) {
Log.v(TAG, "My oncreate running");
resourceManager = new ResourceManager(this);
Log.v(TAG, "Ready to unpack");
File app_root_file = new File(getAppRoot());
unpackData("private", app_root_file);
Log.v(TAG, "About to do super onCreate");
super.onCreate(savedInstanceState);
Log.v(TAG, "Did super onCreate");
this.mActivity = this;
//this.showLoadingScreen();
Log.v("Python", "Device: " + android.os.Build.DEVICE);
Log.v("Python", "Model: " + android.os.Build.MODEL);
//Log.v(TAG, "Ready to unpack");
//new UnpackFilesTask().execute(getAppRoot());
PythonActivity.initialize();
// Load shared libraries
String errorMsgBrokenLib = "";
try {
loadLibraries();
} catch(UnsatisfiedLinkError e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
} catch(Exception e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
}
if (mBrokenLibraries)
{
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
+ System.getProperty("line.separator")
+ System.getProperty("line.separator")
+ "Error: " + errorMsgBrokenLib);
dlgAlert.setTitle("Python Error");
dlgAlert.setPositiveButton("Exit",
new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog,int id) {
// if this button is clicked, close current activity
PythonActivity.mActivity.finish();
}
});
dlgAlert.setCancelable(false);
dlgAlert.create().show();
return;
}
// Set up the Python environment
String app_root_dir = getAppRoot();
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
Log.v(TAG, "Setting env vars for start.c and Python to use");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
try {
Log.v(TAG, "Access to our meta-data...");
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
mActivity.mWakeLock.acquire();
}
} catch (PackageManager.NameNotFoundException e) {
}
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
PythonActivity.mPythonThread = pythonThread;
pythonThread.start();
}
@Override
public void onDestroy() {
Log.i("Destroy", "end of app");
super.onDestroy();
// make sure all child threads (python_thread) are stopped
android.os.Process.killProcess(android.os.Process.myPid());
}
public void loadLibraries() {
String app_root = new String(getAppRoot());
File app_root_file = new File(app_root);
PythonUtil.loadLibraries(app_root_file);
}
public void recursiveDelete(File f) {
if (f.isDirectory()) {
for (File r : f.listFiles()) {
recursiveDelete(r);
}
}
f.delete();
}
/**
* Show an error using a toast. (Only makes sense from non-UI
* threads.)
*/
public void toastError(final String msg) {
final Activity thisActivity = this;
runOnUiThread(new Runnable () {
public void run() {
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
}
});
// Wait to show the error.
synchronized (this) {
try {
this.wait(1000);
} catch (InterruptedException e) {
}
}
}
public void unpackData(final String resource, File target) {
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
// The version of data in memory and on disk.
String data_version = resourceManager.getString(resource + "_version");
String disk_version = null;
Log.v(TAG, "Data version is " + data_version);
// If no version, no unpacking is necessary.
if (data_version == null) {
return;
}
// Check the current disk version, if any.
String filesDir = target.getAbsolutePath();
String disk_version_fn = filesDir + "/" + resource + ".version";
try {
byte buf[] = new byte[64];
InputStream is = new FileInputStream(disk_version_fn);
int len = is.read(buf);
disk_version = new String(buf, 0, len);
is.close();
} catch (Exception e) {
disk_version = "";
}
// If the disk data is out of date, extract it and write the
// version file.
// if (! data_version.equals(disk_version)) {
if (! data_version.equals(disk_version)) {
Log.v(TAG, "Extracting " + resource + " assets.");
recursiveDelete(target);
target.mkdirs();
AssetExtract ae = new AssetExtract(this);
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
toastError("Could not extract " + resource + " data.");
}
try {
// Write .nomedia.
new File(target, ".nomedia").createNewFile();
// Write version file.
FileOutputStream os = new FileOutputStream(disk_version_fn);
os.write(data_version.getBytes());
os.close();
} catch (Exception e) {
Log.w("python", e);
}
}
}
long lastBackClick = SystemClock.elapsedRealtime();
@Override
public boolean onKeyDown(int keyCode, KeyEvent event) {
// If it wasn't the Back key or there's no web page history, bubble up to the default
// system behavior (probably exit the activity)
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
lastBackClick = SystemClock.elapsedRealtime();
Toast.makeText(this, "Click again to close the app",
Toast.LENGTH_LONG).show();
return true;
}
lastBackClick = SystemClock.elapsedRealtime();
return super.onKeyDown(keyCode, event);
}
//----------------------------------------------------------------------------
// Listener interface for onNewIntent
//
public interface NewIntentListener {
void onNewIntent(Intent intent);
}
private List<NewIntentListener> newIntentListeners = null;
public void registerNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
this.newIntentListeners.add(listener);
}
public void unregisterNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
return;
this.newIntentListeners.remove(listener);
}
@Override
protected void onNewIntent(Intent intent) {
if ( this.newIntentListeners == null )
return;
this.onResume();
synchronized ( this.newIntentListeners ) {
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
while ( iterator.hasNext() ) {
(iterator.next()).onNewIntent(intent);
}
}
}
//----------------------------------------------------------------------------
// Listener interface for onActivityResult
//
public interface ActivityResultListener {
void onActivityResult(int requestCode, int resultCode, Intent data);
}
private List<ActivityResultListener> activityResultListeners = null;
public void registerActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
this.activityResultListeners.add(listener);
}
public void unregisterActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
return;
this.activityResultListeners.remove(listener);
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
if ( this.activityResultListeners == null )
return;
this.onResume();
synchronized ( this.activityResultListeners ) {
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
while ( iterator.hasNext() )
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
}
}
public static void start_service(String serviceTitle, String serviceDescription,
String pythonServiceArgument) {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
serviceIntent.putExtra("serviceTitle", serviceTitle);
serviceIntent.putExtra("serviceDescription", serviceDescription);
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
PythonActivity.mActivity.startService(serviceIntent);
}
public static void stop_service() {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
PythonActivity.mActivity.stopService(serviceIntent);
}
public static native void nativeSetEnv(String j_name, String j_value);
public static native int nativeInit(Object arguments);
}
class PythonMain implements Runnable {
@Override
public void run() {
PythonActivity.nativeInit(new String[0]);
}
}
@@ -0,0 +1,45 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonEvent {
private final Lock lock = new ReentrantLock();
private final Condition cond = lock.newCondition();
private boolean flag = false;
public void set() {
lock.lock();
try {
flag = true;
cond.signalAll();
} finally {
lock.unlock();
}
}
public void wait_() throws InterruptedException {
lock.lock();
try {
while (!flag) {
cond.await();
}
} finally {
lock.unlock();
}
}
public void clear() {
lock.lock();
try {
flag = false;
cond.signalAll();
} finally {
lock.unlock();
}
}
}
@@ -0,0 +1,19 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonLock {
private final Lock lock = new ReentrantLock();
public void acquire() {
lock.lock();
}
public void release() {
lock.unlock();
}
}
@@ -0,0 +1,257 @@
package org.renpy.android;
import java.util.List;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
import android.hardware.Sensor;
import android.hardware.SensorEvent;
import android.hardware.SensorEventListener;
import android.hardware.SensorManager;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import android.net.wifi.ScanResult;
import android.net.wifi.WifiManager;
import android.os.Vibrator;
import android.view.View;
/**
* Methods that are expected to be called via JNI, to access the device's
* non-screen hardware. (For example, the vibration and accelerometer.)
*/
public class Hardware {
// The context.
static Context context;
static View view;
/**
* Vibrate for s seconds.
*/
public static void vibrate(double s) {
Vibrator v = (Vibrator) context
.getSystemService(Context.VIBRATOR_SERVICE);
if (v != null) {
v.vibrate((int) (1000 * s));
}
}
/**
* Get an Overview of all Hardware Sensors of an Android Device
*/
public static String getHardwareSensors() {
SensorManager sm = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
if (allSensors != null) {
String resultString = "";
for (Sensor s : allSensors) {
resultString += String.format("Name=" + s.getName());
resultString += String.format(",Vendor=" + s.getVendor());
resultString += String.format(",Version=" + s.getVersion());
resultString += String.format(",MaximumRange="
+ s.getMaximumRange());
// XXX MinDelay is not in the 2.2
// resultString += String.format(",MinDelay=" +
// s.getMinDelay());
resultString += String.format(",Power=" + s.getPower());
resultString += String.format(",Type=" + s.getType() + "\n");
}
return resultString;
}
return "";
}
/**
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
* Magnetic Field Sensors
*/
public static class generic3AxisSensor implements SensorEventListener {
private final SensorManager sSensorManager;
private final Sensor sSensor;
private final int sSensorType;
SensorEvent sSensorEvent;
public generic3AxisSensor(int sensorType) {
sSensorType = sensorType;
sSensorManager = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
sSensor = sSensorManager.getDefaultSensor(sSensorType);
}
public void onAccuracyChanged(Sensor sensor, int accuracy) {
}
public void onSensorChanged(SensorEvent event) {
sSensorEvent = event;
}
/**
* Enable or disable the Sensor by registering/unregistering
*/
public void changeStatus(boolean enable) {
if (enable) {
sSensorManager.registerListener(this, sSensor,
SensorManager.SENSOR_DELAY_NORMAL);
} else {
sSensorManager.unregisterListener(this, sSensor);
}
}
/**
* Read the Sensor
*/
public float[] readSensor() {
if (sSensorEvent != null) {
return sSensorEvent.values;
} else {
float rv[] = { 0f, 0f, 0f };
return rv;
}
}
}
public static generic3AxisSensor accelerometerSensor = null;
public static generic3AxisSensor orientationSensor = null;
public static generic3AxisSensor magneticFieldSensor = null;
/**
* functions for backward compatibility reasons
*/
public static void accelerometerEnable(boolean enable) {
if (accelerometerSensor == null)
accelerometerSensor = new generic3AxisSensor(
Sensor.TYPE_ACCELEROMETER);
accelerometerSensor.changeStatus(enable);
}
public static float[] accelerometerReading() {
float rv[] = { 0f, 0f, 0f };
if (accelerometerSensor == null)
return rv;
return (float[]) accelerometerSensor.readSensor();
}
public static void orientationSensorEnable(boolean enable) {
if (orientationSensor == null)
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
orientationSensor.changeStatus(enable);
}
public static float[] orientationSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (orientationSensor == null)
return rv;
return (float[]) orientationSensor.readSensor();
}
public static void magneticFieldSensorEnable(boolean enable) {
if (magneticFieldSensor == null)
magneticFieldSensor = new generic3AxisSensor(
Sensor.TYPE_MAGNETIC_FIELD);
magneticFieldSensor.changeStatus(enable);
}
public static float[] magneticFieldSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (magneticFieldSensor == null)
return rv;
return (float[]) magneticFieldSensor.readSensor();
}
/**
* Scan WiFi networks
*/
static List<ScanResult> latestResult;
public static void enableWifiScanner() {
IntentFilter i = new IntentFilter();
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
// occurs
WifiManager w = (WifiManager) c
.getSystemService(Context.WIFI_SERVICE);
latestResult = w.getScanResults(); // Returns a <list> of
// scanResults
}
}, i);
}
public static String scanWifi() {
// Now you can call this and it should execute the broadcastReceiver's
// onReceive()
WifiManager wm = (WifiManager) context
.getSystemService(Context.WIFI_SERVICE);
boolean a = wm.startScan();
if (latestResult != null) {
String latestResultString = "";
for (ScanResult result : latestResult) {
latestResultString += String.format("%s\t%s\t%d\n",
result.SSID, result.BSSID, result.level);
}
return latestResultString;
}
return "";
}
/**
* network state
*/
public static boolean network_state = false;
/**
* Check network state directly
*
* (only one connection can be active at a given moment, detects all network
* type)
*
*/
public static boolean checkNetwork() {
boolean state = false;
final ConnectivityManager conMgr = (ConnectivityManager) context
.getSystemService(Context.CONNECTIVITY_SERVICE);
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
if (activeNetwork != null && activeNetwork.isConnected()) {
state = true;
} else {
state = false;
}
return state;
}
/**
* To recieve network state changes
*/
public static void registerNetworkCheck() {
IntentFilter i = new IntentFilter();
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
network_state = checkNetwork();
}
}, i);
}
}
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,93 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="{{ args.package }}"
android:versionCode="{{ args.numeric_version }}"
android:versionName="{{ args.version }}"
android:installLocation="auto">
<supports-screens
android:smallScreens="true"
android:normalScreens="true"
android:largeScreens="true"
android:anyDensity="true"
{% if args.min_sdk_version >= 9 %}
android:xlargeScreens="true"
{% endif %}
/>
<!-- Android 2.3.3 -->
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
<!-- Set permissions -->
{% for perm in args.permissions %}
{% if '.' in perm %}
<uses-permission android:name="{{ perm }}" />
{% else %}
<uses-permission android:name="android.permission.{{ perm }}" />
{% endif %}
{% endfor %}
{% if args.wakelock %}
<uses-permission android:name="android.permission.WAKE_LOCK" />
{% endif %}
{% if args.billing_pubkey %}
<uses-permission android:name="com.android.vending.BILLING" />
{% endif %}
<!-- Create a Java class extending SDLActivity and place it in a
directory under src matching the package, e.g.
src/com/gamemaker/game/MyGame.java
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
in the XML below.
An example Java class can be found in README-android.txt
-->
<application android:label="@string/app_name"
android:icon="@drawable/icon"
android:allowBackup="true"
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
android:hardwareAccelerated="true" >
{% for m in args.meta_data %}
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
<activity android:name="org.kivy.android.PythonActivity"
android:label="@string/app_name"
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
>
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
{%- if args.intent_filters -%}
{{- args.intent_filters -}}
{%- endif -%}
</activity>
{% if service %}
<service android:name="org.kivy.android.PythonService"
android:process=":pythonservice" />
{% endif %}
{% for name in service_names %}
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
android:process=":service_{{ name }}" />
{% endfor %}
{% if args.billing_pubkey %}
<service android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbilling" />
<receiver android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbillingreceiver">
<intent-filter>
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
</intent-filter>
</receiver>
{% endif %}
</application>
</manifest>
@@ -0,0 +1,74 @@
package {{ args.package }};
import android.os.Binder;
import android.os.IBinder;
import android.content.Intent;
import android.content.Context;
import org.kivy.android.PythonService;
public class Service{{ name|capitalize }} extends PythonService {
/**
* Binder given to clients
*/
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
{% if sticky %}
/**
* {@inheritDoc}
*/
@Override
public int getStartType() {
return START_STICKY;
}
{% endif %}
{% if foreground %}
/**
* {@inheritDoc}
*/
@Override
public boolean getStartForeground() {
return true;
}
{% endif %}
public static void start(Context ctx, String pythonServiceArgument) {
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
intent.putExtra("androidPrivate", argument);
intent.putExtra("androidArgument", argument);
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
intent.putExtra("serviceDescription", "");
intent.putExtra("pythonName", "{{ name }}");
intent.putExtra("pythonHome", argument);
intent.putExtra("androidUnpack", argument);
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
ctx.startService(intent);
}
public static void stop(Context ctx) {
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
ctx.stopService(intent);
}
/**
* {@inheritDoc}
*/
@Override
public IBinder onBind(Intent intent) {
return mBinder;
}
/**
* Class used for the client Binder. Because we know this service always
* runs in the same process as its clients, we don't need to deal with IPC.
*/
public class Service{{ name|capitalize }}Binder extends Binder {
Service{{ name|capitalize }} getService() {
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
return Service{{ name|capitalize }}.this;
}
}
}
@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string name="app_name">{{ args.name }}</string>
<string name="private_version">{{ private_version }}</string>
</resources>
+68
View File
@@ -0,0 +1,68 @@
from lbrynet.extras.daemon.Daemon import Daemon
from jnius import PythonJavaClass, autoclass, java_method
JAVA_NAMESPACE = 'org.kivy.android'
JNI_NAMESPACE = 'org/kivy/android'
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
_callbacks = {
'on_new_intent': [],
'on_activity_result': [],
}
class NewIntentListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
__javacontext__ = 'app'
def __init__(self, callback, **kwargs):
super(NewIntentListener, self).__init__(**kwargs)
self.callback = callback
@java_method('(Landroid/content/Intent;)V')
def onNewIntent(self, intent):
self.callback(intent)
class ActivityResultListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
__javacontext__ = 'app'
def __init__(self, callback):
super(ActivityResultListener, self).__init__()
self.callback = callback
@java_method('(IILandroid/content/Intent;)V')
def onActivityResult(self, requestCode, resultCode, intent):
self.callback(requestCode, resultCode, intent)
def bind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
elif event == 'on_new_intent':
listener = NewIntentListener(callback)
_activity.registerNewIntentListener(listener)
_callbacks[event].append(listener)
elif event == 'on_activity_result':
listener = ActivityResultListener(callback)
_activity.registerActivityResultListener(listener)
_callbacks[event].append(listener)
def unbind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
else:
for listener in _callbacks[event][:]:
if listener.callback == callback:
_callbacks[event].remove(listener)
if event == 'on_new_intent':
_activity.unregisterNewIntentListener(listener)
elif event == 'on_activity_result':
_activity.unregisterActivityResultListener(listener)
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
+49
View File
@@ -0,0 +1,49 @@
from os.path import join, dirname, abspath
from pythonforandroid.toolchain import Bootstrap
from setuptools import setup, find_packages
from bootstrap import LBRYServiceBootstrap
Bootstrap.bootstraps = {
'lbry-service': LBRYServiceBootstrap()
}
setup(
name='lbryservice',
version='0.1',
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
description="Android Service for LBRY Network.",
license='MIT',
python_requires='>=3.7',
packages=find_packages(),
package_data={'service': ['*.py']},
options={
'apk': {
'dist_name': 'lbry-service',
'bootstrap': 'lbry-service',
'package': 'io.lbry.service',
'permissions': ['INTERNET'],
'requirements': ','.join([
# needed by aiohttp
'multidict', 'yarl', 'async_timeout', 'chardet',
# minimum needed by torba:
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
abspath(join(dirname(__file__), '..', '..', 'torba')),
# minimum needed by lbrynet
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
abspath(join(dirname(__file__), '..')),
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
]),
'android-api': '26',
'ndk-api': '21',
'ndk-version': 'r17c',
'arch': 'armeabi-v7a',
'sdk-dir': '/home/lex/projects/android',
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
}
}
)
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.31.0rc5"
__version__ = "0.31.0rc3"
version = tuple(__version__.split('.'))
+8 -8
View File
@@ -77,7 +77,7 @@ class BlobFile:
def writer_finished(self, writer: HashBlobWriter):
def callback(finished: asyncio.Future):
try:
error = finished.exception()
error = finished.result()
except Exception as err:
error = err
if writer in self.writers: # remove this download attempt
@@ -86,7 +86,7 @@ class BlobFile:
while self.writers:
other = self.writers.pop()
other.finished.cancel()
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
t = self.loop.create_task(self.save_verified_blob(writer))
t.add_done_callback(lambda *_: self.finished_writing.set())
return
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
@@ -96,24 +96,24 @@ class BlobFile:
raise error
return callback
async def save_verified_blob(self, writer, verified_bytes: bytes):
async def save_verified_blob(self, writer):
def _save_verified():
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
if self.get_length() == len(verified_bytes):
if self.get_length() == len(writer.verified_bytes):
with open(self.file_path, 'wb') as write_handle:
write_handle.write(verified_bytes)
write_handle.write(writer.verified_bytes)
self.saved_verified_blob = True
else:
raise Exception("length mismatch")
if self.verified.is_set():
return
async with self.blob_write_lock:
if self.verified.is_set():
return
await self.loop.run_in_executor(None, _save_verified)
self.verified.set()
if self.blob_completed_callback:
await self.blob_completed_callback(self)
self.verified.set()
def open_for_writing(self) -> HashBlobWriter:
if os.path.exists(self.file_path):
+8 -4
View File
@@ -18,6 +18,7 @@ class HashBlobWriter:
self.finished.add_done_callback(lambda *_: self.close_handle())
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.verified_bytes = b''
def __del__(self):
if self.buffer is not None:
@@ -45,7 +46,7 @@ class HashBlobWriter:
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.close_handle()
self.finished.set_exception(InvalidDataError(
self.finished.set_result(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
return
@@ -54,12 +55,15 @@ class HashBlobWriter:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.close_handle()
self.finished.set_exception(InvalidBlobHashError(
self.finished.set_result(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(self.buffer.getvalue())
return
self.buffer.seek(0)
self.verified_bytes = self.buffer.read()
self.close_handle()
if self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(None)
def close_handle(self):
if self.buffer is not None:
+54 -63
View File
@@ -1,7 +1,6 @@
import asyncio
import logging
import typing
import binascii
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
@@ -20,15 +19,15 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None
self.download_running = asyncio.Event(loop=self.loop)
self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None
self._request_lock = asyncio.Lock(loop=self.loop)
def data_received(self, data: bytes):
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
if self._response_fut and not self._response_fut.done():
def handle_data_received(self, data: bytes):
if self.transport.is_closing():
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
self._response_fut.cancel()
return
@@ -37,74 +36,60 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
if response.responses and self.blob:
blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
# set the expected length for the incoming blob if we didn't know it
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return
if response.responses:
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
# fire the Future with the response to our request
self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed():
log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
# write blob bytes if we're writing a blob and have blob bytes to write
self._blob_bytes_received += len(response.blob_data)
try:
self.writer.write(response.blob_data)
return
except IOError as err:
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
except (asyncio.CancelledError, asyncio.TimeoutError) as err: # TODO: is this needed?
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
log.error("error downloading blob: %s", err)
def data_received(self, data):
try:
return self.handle_data_received(data)
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[bool, bool]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
self.transport.write(request.serialize())
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return False, False
log.warning("blob not in availability response")
return False, True
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
log.warning("blob availability response doesn't match our request")
return False, False
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
log.warning("data rate rejected")
return False, False
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
log.warning("blob cant be downloaded from this peer")
return False, True
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
log.warning("incoming blob hash mismatch")
return False, False
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
log.warning("incoming blob unexpected length")
return False, False
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
log.info(msg)
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
await self.blob.finished_writing.wait()
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
log.info(msg)
return True, True
except asyncio.CancelledError:
@@ -119,6 +104,9 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self._response_fut.cancel()
if self.writer and not self.writer.closed():
self.writer.close_handle()
if self.blob:
await self.blob.close()
self.download_running.clear()
self._response_fut = None
self.writer = None
self.blob = None
@@ -129,49 +117,52 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
if blob.get_is_verified():
return False, True
try:
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError:
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
# i'm not sure how to fix this race condition - jack
return False, True
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, False
except asyncio.CancelledError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, True
async with self._request_lock:
try:
if self.download_running.is_set():
log.info("wait for download already running")
await self.download_running.wait()
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self.download_running.set()
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError:
log.error("race happened")
# i'm not sure how to fix this race condition - jack
return False, True
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, False
except asyncio.CancelledError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, True
def connection_made(self, transport: asyncio.Transport):
self.transport = transport
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
# log.info("connection made to %s: %s", self.peer_address, transport)
def connection_lost(self, reason):
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
self.transport = None
self.loop.create_task(self.close())
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float) -> typing.Tuple[bool, bool]:
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
if blob.get_is_verified():
return False, True
if blob.get_is_verified():
log.info("already verified")
return False, True
try:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
return await protocol.download_blob(blob)
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return False, False
finally:
await protocol.close()
+104 -83
View File
@@ -1,10 +1,10 @@
import asyncio
import typing
import logging
from lbrynet import conf
from lbrynet.utils import drain_tasks
from lbrynet.blob_exchange.client import request_blob
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
@@ -18,94 +18,115 @@ def drain_into(a: list, b: list):
b.append(a.pop())
class BlobDownloader:
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
peer_queue: asyncio.Queue):
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
"""A single blob downloader"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.peer_queue = peer_queue
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Set['KademliaPeer'] = set()
self.scores: typing.Dict['KademliaPeer', int] = {}
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.lock = asyncio.Lock(loop=self.loop)
self.blob: 'BlobFile' = None
self.blob_queue = asyncio.Queue(loop=self.loop)
def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
async def _request_blob():
if blob.get_is_verified():
return
try:
success, keep_connection = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout
)
finally:
if peer in self.active_connections:
self.active_connections.pop(peer)
if not keep_connection and peer not in self.ignored:
self.ignored.add(peer)
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
elif keep_connection:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
if success:
self.scores[peer] = self.scores.get(peer, 0) + 2
else:
self.scores[peer] = self.scores.get(peer, 0) - 1
return self.loop.create_task(_request_blob())
self.blob_download_timeout = config.get('blob_download_timeout')
self.peer_connect_timeout = config.get('peer_connect_timeout')
self.max_connections = config.get('max_connections_per_stream')
async def new_peer_or_finished(self, blob: 'BlobFile'):
async def get_and_re_add_peers():
new_peers = await self.peer_queue.get()
self.peer_queue.put_nowait(new_peers)
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
async def _request_blob(self, peer: 'KademliaPeer'):
if self.blob.get_is_verified():
log.info("already verified")
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self.lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
def _update_requests(self):
self.new_peer_event.clear()
if self.blob.blob_hash not in self.requested_from:
self.requested_from[self.blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.blob.blob_hash][peer] = task
self.running_download_requests.append(task)
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
async def _accumulate_connections(self, node: 'Node'):
try:
await asyncio.wait(tasks, loop=self.loop, return_when='FIRST_COMPLETED')
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
return
except asyncio.CancelledError:
drain_tasks(tasks)
raise
pass
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
self.blob = self.blob_manager.get_blob(blob_hash)
if self.blob.get_is_verified():
return self.blob
accumulator = self.loop.create_task(self._accumulate_connections(node))
self.blob_queue.put_nowait(blob_hash)
try:
while not blob.get_is_verified():
batch: typing.List['KademliaPeer'] = []
while not self.peer_queue.empty():
batch.extend(await self.peer_queue.get())
for peer in batch:
if len(self.active_connections) >= self.config.max_connections_per_download:
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
self.active_connections[peer] = self.request_blob_from_peer(blob, peer)
await self.new_peer_or_finished(blob)
to_re_add = list(set(filter(lambda peer: peer not in self.ignored, batch)))
to_re_add.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
if to_re_add:
self.peer_queue.put_nowait(to_re_add)
while self.active_connections:
peer, task = self.active_connections.popitem()
if task and not task.done():
task.cancel()
await blob.close()
log.info("downloaded %s", blob_hash[:8])
return blob
while not self.blob.get_is_verified():
if len(self.running_download_requests) < self.max_connections:
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
if self.blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.blob
except asyncio.CancelledError:
while self.active_connections:
peer, task = self.active_connections.popitem()
if task and not task.done():
task.cancel()
drain_tasks(self.running_download_requests)
raise
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
blob_hash: str) -> 'BlobFile':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
try:
return await downloader.download_blob(blob_hash)
finally:
if accumulate_task and not accumulate_task.done():
accumulate_task.cancel()
finally:
if accumulator and not accumulator.done():
accumulator.cancel()
+18 -12
View File
@@ -146,26 +146,32 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
# <blob bytes>
# <json><blob bytes>
extra_data = b''
response = None
curr_pos = 0
while True:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren == -1:
return None, response_msg
break
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
if not isinstance(response, dict):
raise ValueError()
for key in response.keys():
if key not in [
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key]:
raise ValueError()
extra_data = response_msg[curr_pos:]
break
except ValueError:
continue
possible_response_keys = {
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
}
if isinstance(response, dict) and response.keys():
if set(response.keys()).issubset(possible_response_keys):
return response, response_msg[curr_pos:]
return None, response_msg
response = None
if response is None:
extra_data = response_msg
return response, extra_data
class BlobRequest:
+3 -8
View File
@@ -57,13 +57,8 @@ class BlobServerProtocol(asyncio.Protocol):
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
try:
sent = await blob.sendfile(self)
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
if self.transport:
self.transport.close()
return
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
sent = await blob.sendfile(self)
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
if responses:
self.send_response(responses)
@@ -77,7 +72,7 @@ class BlobServerProtocol(asyncio.Protocol):
self.buf += data
return
try:
request = BlobRequest.deserialize(self.buf + data)
request = BlobRequest.deserialize(data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
+64 -63
View File
@@ -16,11 +16,29 @@ log = logging.getLogger(__name__)
NOT_SET = type(str('NOT_SET'), (object,), {})
T = typing.TypeVar('T')
KB = 2 ** 10
MB = 2 ** 20
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
API_ADDRESS = 'lbryapi'
APP_NAME = 'LBRY'
BLOBFILES_DIR = 'blobfiles'
CRYPTSD_FILE_EXTENSION = '.cryptsd'
CURRENCIES = {
'BTC': {'type': 'crypto'},
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
LOG_FILE_NAME = 'lbrynet.log'
LOG_POST_URL = 'https://lbry.io/log-upload'
MAX_BLOB_REQUEST_SIZE = 64 * KB
MAX_HANDSHAKE_SIZE = 64 * KB
MAX_REQUEST_SIZE = 64 * KB
MAX_RESPONSE_INFO_SIZE = 64 * KB
MAX_BLOB_INFOS_TO_REQUEST = 20
PROTOCOL_PREFIX = 'lbry'
SLACK_WEBHOOK = (
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
@@ -453,7 +471,7 @@ class CLIConfig(BaseConfig):
class Config(CLIConfig):
# directories
data_dir = Path("Directory path to store blobs.", metavar='DIR')
download_dir = Path(
"Directory path to place assembled files downloaded from LBRY.",
@@ -464,77 +482,60 @@ class Config(CLIConfig):
previous_names=['lbryum_wallet_dir'], metavar='DIR'
)
# network
use_upnp = Toggle(
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
"ports or have firewall rules you likely want to disable this.", True
)
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 20.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
# blob announcement and download
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
)
concurrent_blob_announcers = Integer(
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
previous_names=['concurrent_announcers']
)
max_connections_per_download = Integer(
"Maximum number of peers to connect to while downloading a blob", 8,
previous_names=['max_connections_per_stream']
)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
)
max_key_fee = MaxKeyFee(
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
) # TODO: use this
# reflector settings
reflect_streams = Toggle(
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
previous_names=['reflect_uploads']
)
concurrent_reflector_uploads = Integer(
"Maximum number of streams to upload to a reflector server at a time", 10
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
# claims set to expire within this many blocks will be
# automatically renewed after startup (if set to 0, renews
# will not be made automatically)
auto_renew_claim_height_delta = Integer("", 0)
cache_time = Integer("", 150)
data_rate = Float("points/megabyte", .0001)
delete_blobs_on_remove = Toggle("", True)
dht_node_port = Integer("", 4444)
download_timeout = Float("", 30.0)
blob_download_timeout = Float("", 20.0)
peer_connect_timeout = Float("", 3.0)
node_rpc_timeout = Float("", constants.rpc_timeout)
is_generous_host = Toggle("", True)
announce_head_blobs_only = Toggle("", True)
concurrent_announcers = Integer("", 10)
known_dht_nodes = Servers("", [
('lbrynet1.lbry.io', 4444), # US EAST
('lbrynet2.lbry.io', 4444), # US WEST
('lbrynet3.lbry.io', 4444), # EU
('lbrynet4.lbry.io', 4444) # ASIA
])
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
max_connections_per_stream = Integer("", 5)
seek_head_blob_first = Toggle("", True)
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
min_info_rate = Float("points/1000 infos", .02)
min_valuable_hash_rate = Float("points/1000 infos", .05)
min_valuable_info_rate = Float("points/1000 infos", .05)
peer_port = Integer("", 3333)
pointtrader_server = String("", 'http://127.0.0.1:2424')
reflector_port = Integer("", 5566)
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
reflect_uploads = Toggle("", True)
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
reflector_servers = Servers("", [
('reflector.lbry.io', 5566)
])
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
sd_download_timeout = Integer("", 3)
peer_search_timeout = Integer("", 60)
use_upnp = Toggle("", True)
use_keyring = Toggle("", False)
blockchain_name = String("", 'lbrycrd_main')
lbryum_servers = Servers("", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
def __init__(self, **kwargs):
super().__init__(**kwargs)
+1 -3
View File
@@ -19,8 +19,6 @@ class BlobAnnouncer:
self.announce_queue: typing.List[str] = []
async def _announce(self, batch_size: typing.Optional[int] = 10):
if not batch_size:
return
if not self.node.joined.is_set():
await self.node.joined.wait()
blob_hashes = await self.storage.get_blobs_to_announce()
@@ -41,7 +39,7 @@ class BlobAnnouncer:
to_await.append(batch.pop())
if to_await:
await asyncio.gather(*tuple(to_await), loop=self.loop)
await self.storage.update_last_announced_blobs(announced)
await self.storage.update_last_announced_blobs(announced, self.loop.time())
log.info("announced %i blobs", len(announced))
if self.running:
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
+32 -55
View File
@@ -1,9 +1,9 @@
import logging
import asyncio
import typing
import socket
import binascii
import contextlib
from lbrynet.utils import resolve_host
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
@@ -20,10 +20,10 @@ log = logging.getLogger(__name__)
class Node:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
internal_udp_port: int, peer_port: int, external_ip: str):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
@@ -79,9 +79,10 @@ class Node:
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.debug("Store to %i peers", len(peers))
log.info("Store to %i peers", len(peers))
log.info(peers)
for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
@@ -115,50 +116,39 @@ class Node:
log.warning("Already bound to port %s", self.listening_port)
async def join_network(self, interface: typing.Optional[str] = '',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
known_node_addresses = known_node_addresses or []
if known_node_urls:
for host, port in known_node_urls:
address = await resolve_host(host)
if (address, port) not in known_node_addresses:
known_node_addresses.append((address, port))
url_to_addr[address] = host
info = await self.loop.getaddrinfo(
host, 'https',
proto=socket.IPPROTO_TCP,
)
if (info[0][4][0], port) not in known_node_addresses:
known_node_addresses.append((info[0][4][0], port))
futs = []
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
futs.append(peer.ping())
if futs:
await asyncio.wait(futs, loop=self.loop)
if known_node_addresses:
while not self.protocol.routing_table.get_peers():
success = False
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
for address, port in known_node_addresses:
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
try:
await peer.ping()
success = True
except asyncio.TimeoutError:
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
round(self.protocol.rpc_timeout, 2))
if success:
break
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
# in the routing table with good peers who are near us
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
async for peers in junction:
for peer in peers:
try:
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
await self.protocol.get_rpc_peer(peer).ping()
except (asyncio.TimeoutError, RemoteException):
pass
self.joined.set()
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
@@ -224,26 +214,13 @@ class Node:
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
bottom_out_limit=20) -> typing.List['KademliaPeer']:
accumulated: typing.List['KademliaPeer'] = []
async with self.peer_search_junction(node_id, max_results=max_results,
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
bottom_out_limit=bottom_out_limit) as junction:
async for peers in junction:
log.info("peer search: %s", peers)
accumulated.extend(peers)
log.info("junction done")
log.info("context done")
distance = Distance(node_id)
accumulated.sort(key=lambda peer: distance(peer.node_id))
return accumulated[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
try:
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
async for peers in search_junction:
if peers:
result_queue.put_nowait(peers)
except asyncio.CancelledError:
return
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue()
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
+54 -33
View File
@@ -30,30 +30,37 @@ class PeerManager:
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
self._lock = asyncio.Lock(loop=loop)
def report_failure(self, address: str, udp_port: int):
async def report_failure(self, address: str, udp_port: int):
now = self._loop.time()
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
async with self._lock:
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
def report_last_sent(self, address: str, udp_port: int):
async def report_last_sent(self, address: str, udp_port: int):
now = self._loop.time()
self._last_sent[(address, udp_port)] = now
async with self._lock:
self._last_sent[(address, udp_port)] = now
def report_last_replied(self, address: str, udp_port: int):
async def report_last_replied(self, address: str, udp_port: int):
now = self._loop.time()
self._last_replied[(address, udp_port)] = now
async with self._lock:
self._last_replied[(address, udp_port)] = now
def report_last_requested(self, address: str, udp_port: int):
async def report_last_requested(self, address: str, udp_port: int):
now = self._loop.time()
self._last_requested[(address, udp_port)] = now
async with self._lock:
self._last_requested[(address, udp_port)] = now
def clear_token(self, node_id: bytes):
self._node_tokens.pop(node_id, None)
async def clear_token(self, node_id: bytes):
async with self._lock:
self._node_tokens.pop(node_id, None)
def update_token(self, node_id: bytes, token: bytes):
async def update_token(self, node_id: bytes, token: bytes):
now = self._loop.time()
self._node_tokens[node_id] = (now, token)
async with self._lock:
self._node_tokens[node_id] = (now, token)
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
ts, token = self._node_tokens.get(node_id, (None, None))
@@ -63,36 +70,50 @@ class PeerManager:
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
return self._last_replied.get((address, udp_port))
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
return self._node_id_mapping.get((address, udp_port))
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
return self._node_id_reverse_mapping.get(node_id)
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
async with self._lock:
addr_tuple = self._node_id_reverse_mapping.get(node_id)
if addr_tuple and addr_tuple in self._node_id_mapping:
return addr_tuple
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
"""
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
This is to handle peers changing addresses and ids while assuring that the we only ever have
one node id / address tuple mapped to each other
"""
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
async with self._lock:
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
return KademliaPeer(self._loop, address, node_id, udp_port)
def prune(self): # TODO: periodically call this
async def prune(self):
now = self._loop.time()
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
async with self._lock:
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
"""
@@ -100,6 +100,6 @@ class AsyncGeneratorJunction:
pass
finally:
if exc_type:
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
err = traceback.format_exception(exc_type, exc, tb)
log.error(err)
+116 -84
View File
@@ -1,9 +1,7 @@
import asyncio
from binascii import hexlify
from itertools import chain
import typing
import logging
from lbrynet.utils import drain_tasks
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException
from lbrynet.dht.protocol.distance import Distance
@@ -92,8 +90,8 @@ class IterativeFinder:
self.exclude = exclude or []
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
self.active: typing.Set['KademliaPeer'] = set()
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
self.active: typing.List['KademliaPeer'] = []
self.contacted: typing.List[typing.Tuple[str, int]] = []
self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
@@ -101,12 +99,14 @@ class IterativeFinder:
self.iteration_queue = asyncio.Queue(loop=self.loop)
self.running_probes: typing.Set[asyncio.Task] = set()
self.running_probes: typing.List[asyncio.Task] = []
self.lock = asyncio.Lock(loop=self.loop)
self.iteration_count = 0
self.bottom_out_count = 0
self.running = False
self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = []
self.finished = asyncio.Event(loop=self.loop)
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
"""
@@ -114,16 +114,9 @@ class IterativeFinder:
"""
raise NotImplementedError()
def search_exhausted(self):
"""
This method ends the iterator due no more peers to contact.
Override to provide last time results.
"""
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindResponse):
"""
Called after adding peers from an rpc result to the shortlist.
Called with a lock after adding peers from an rpc result to the shortlist.
This method is responsible for putting a result for the generator into the Queue
"""
raise NotImplementedError()
@@ -136,7 +129,9 @@ class IterativeFinder:
return []
def _is_closer(self, peer: 'KademliaPeer') -> bool:
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
if not self.closest_peer:
return True
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
def _update_closest(self):
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
@@ -146,18 +141,21 @@ class IterativeFinder:
self.closest_peer = self.shortlist[-1]
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.add(peer)
for contact_triple in response.get_close_triples():
node_id, address, udp_port = contact_triple
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self.check_result_ready(response)
async with self.lock:
if peer not in self.shortlist:
self.shortlist.append(peer)
if peer not in self.active:
self.active.append(peer)
for contact_triple in response.get_close_triples():
addr_tuple = (contact_triple[1], contact_triple[2])
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
found_peer = self.peer_manager.get_kademlia_peer(
contact_triple[0], contact_triple[1], contact_triple[2]
)
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
self.shortlist.append(found_peer)
self._update_closest()
self.check_result_ready(response)
async def _send_probe(self, peer: 'KademliaPeer'):
try:
@@ -165,11 +163,13 @@ class IterativeFinder:
except asyncio.CancelledError:
return
except asyncio.TimeoutError:
self.active.discard(peer)
if peer in self.active:
self.active.remove(peer)
return
except ValueError as err:
log.warning(str(err))
self.active.discard(peer)
if peer in self.active:
self.active.remove(peer)
return
except RemoteException:
return
@@ -181,35 +181,31 @@ class IterativeFinder:
"""
added = 0
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.add((peer.address, peer.udp_port))
async with self.lock:
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
while self.running and len(self.shortlist) and added < constants.alpha:
peer = self.shortlist.pop()
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
continue
if peer.node_id == self.protocol.node_id:
continue
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
continue
if (peer.address, peer.udp_port) not in self.contacted:
self.contacted.append((peer.address, peer.udp_port))
t = self.loop.create_task(self._send_probe(peer))
t = self.loop.create_task(self._send_probe(peer))
def callback(_):
self.running_probes.difference_update({
probe for probe in self.running_probes if probe.done() or probe == t
})
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
def callback(_):
if t and t in self.running_probes:
self.running_probes.remove(t)
if not self.running_probes and self.shortlist:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback)
self.running_probes.add(t)
added += 1
log.debug("running %d probes", len(self.running_probes))
if not added and not self.running_probes:
log.debug("search for %s exhausted", hexlify(self.key)[:8])
self.search_exhausted()
t.add_done_callback(callback)
self.running_probes.append(t)
added += 1
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
try:
@@ -219,41 +215,70 @@ class IterativeFinder:
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration):
if self.running:
self.loop.call_soon(self.aclose)
drain_tasks(self.running_probes)
self.running = False
def _search(self):
self.tasks.append(self.loop.create_task(self._search_task()))
def __aiter__(self):
def search(self):
if self.running:
raise Exception("already running")
self.running = True
self._search()
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
peers = self.loop.create_task(self.iteration_queue.get())
finished = self.loop.create_task(self.finished.wait())
err = None
try:
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
if peers.done():
return peers.result()
raise StopAsyncIteration()
except asyncio.CancelledError as error:
err = error
finally:
if not finished.done() and not finished.cancelled():
finished.cancel()
if not peers.done() and not peers.cancelled():
peers.cancel()
if err:
raise err
def __aiter__(self):
self.search()
return self
async def __anext__(self) -> typing.List['KademliaPeer']:
try:
if self.iteration_count == 0:
result = self.get_initial_result() or await self.iteration_queue.get()
else:
result = await self.iteration_queue.get()
if not result:
raise StopAsyncIteration
initial_results = self.get_initial_result()
if initial_results:
self.iteration_queue.put_nowait(initial_results)
result = await self.next_queue_or_finished()
self.iteration_count += 1
return result
except (asyncio.CancelledError, StopAsyncIteration):
self.loop.call_soon(self.aclose)
await self.aclose()
raise
def aclose(self):
self.running = False
self.iteration_queue.put_nowait(None)
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
task.cancel()
self.tasks.clear()
self.running_probes.clear()
self.delayed_calls.clear()
async def _aclose():
async with self.lock:
self.running = False
if not self.finished.is_set():
self.finished.set()
drain_tasks(self.tasks)
drain_tasks(self.running_probes)
while self.delayed_calls:
timer = self.delayed_calls.pop()
if timer:
timer.cancel()
return asyncio.ensure_future(_aclose(), loop=self.loop)
class IterativeNodeFinder(IterativeFinder):
@@ -270,26 +295,24 @@ class IterativeNodeFinder(IterativeFinder):
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
return FindNodeResponse(self.key, response)
def search_exhausted(self):
self.put_result(self.active, finish=True)
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
def put_result(self, from_list: typing.List['KademliaPeer']):
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
if to_yield:
for peer in to_yield:
self.yielded_peers.add(peer)
self.iteration_queue.put_nowait(to_yield)
if finish:
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindNodeResponse):
found = response.found and self.key != self.protocol.node_id
if found:
log.debug("found")
return self.put_result(self.shortlist, finish=True)
log.info("found")
self.put_result(self.shortlist)
if not self.finished.is_set():
self.finished.set()
return
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
@@ -300,10 +323,16 @@ class IterativeNodeFinder(IterativeFinder):
self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active, True)
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.debug("max results")
self.put_result(self.active, True)
self.put_result(self.active)
if not self.finished.is_set():
self.finished.set()
return
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
log.info("max results")
self.put_result(self.active)
if not self.finished.is_set():
self.finished.set()
return
class IterativeValueFinder(IterativeFinder):
@@ -337,11 +366,14 @@ class IterativeValueFinder(IterativeFinder):
# log.info("enough blob peers found")
# if not self.finished.is_set():
# self.finished.set()
elif self.prev_closest_peer and self.closest_peer:
return
if self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
if self.bottom_out_count >= self.bottom_out_limit:
log.info("blob peer search bottomed out")
self.iteration_queue.put_nowait(None)
if not self.finished.is_set():
self.finished.set()
return
def get_initial_result(self) -> typing.List['KademliaPeer']:
if self.protocol.data_store.has_peers_for_blob(self.key):
+14 -14
View File
@@ -86,7 +86,7 @@ class KademliaRPC:
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
peers = []
if has_other_peers:
peers.extend([peer.compact_address_tcp() for peer in self.protocol.data_store.get_peers_for_blob(key)])
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
@@ -180,7 +180,7 @@ class RemoteKademliaRPC:
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
)
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
return response.response
@@ -218,7 +218,7 @@ class PingQueue:
await self._protocol.add_peer(p)
return
await self._protocol.get_rpc_peer(p).ping()
except asyncio.TimeoutError:
except TimeoutError:
pass
while True:
@@ -415,8 +415,8 @@ class KademliaProtocol(DatagramProtocol):
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
# This is an RPC method request
self.peer_manager.report_last_requested(address[0], address[1])
self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
await self.peer_manager.report_last_requested(address[0], address[1])
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
# only add a requesting contact to the routing table if it has replied to one of our requests
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
@@ -457,8 +457,8 @@ class KademliaProtocol(DatagramProtocol):
elif response_datagram.node_id == self.node_id:
df.set_exception(RemoteException("incoming message is from our node id"))
return
self.peer_manager.report_last_replied(address[0], address[1])
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
await self.peer_manager.report_last_replied(address[0], address[1])
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
if not df.cancelled():
df.set_result(response_datagram)
await self.add_peer(peer)
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
try:
message = decode_datagram(datagram)
except (ValueError, TypeError):
self.peer_manager.report_failure(address[0], address[1])
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
return
@@ -522,10 +522,10 @@ class KademliaProtocol(DatagramProtocol):
response_fut = self.sent_messages[request.rpc_id][1]
try:
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
return response
except (asyncio.TimeoutError, RemoteException):
self.peer_manager.report_failure(peer.address, peer.udp_port)
await self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False:
self.routing_table.remove_peer(peer)
raise
@@ -575,9 +575,9 @@ class KademliaProtocol(DatagramProtocol):
else:
raise err
if isinstance(message, RequestDatagram):
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
elif isinstance(message, ErrorDatagram):
self.peer_manager.report_failure(peer.address, peer.udp_port)
await self.peer_manager.report_failure(peer.address, peer.udp_port)
def change_token(self):
self.old_token_secret = self.token_secret
@@ -601,7 +601,7 @@ class KademliaProtocol(DatagramProtocol):
res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK":
raise ValueError(res)
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
return peer.node_id, True
except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
@@ -609,7 +609,7 @@ class KademliaProtocol(DatagramProtocol):
log.error("Unexpected response: %s" % err)
except Exception as err:
if 'Invalid token' in str(err):
self.peer_manager.clear_token(peer.node_id)
await self.peer_manager.clear_token(peer.node_id)
else:
log.exception("Unexpected error while storing blob_hash")
return peer.node_id, False
-11
View File
@@ -1,6 +1,4 @@
import os
import sys
import pathlib
import json
import asyncio
import argparse
@@ -218,19 +216,12 @@ def get_argument_parser():
return main
def ensure_directory_exists(path: str):
if not os.path.isdir(path):
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
def main(argv=None):
argv = argv or sys.argv[1:]
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args)
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
ensure_directory_exists(directory)
if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}")
@@ -252,8 +243,6 @@ def main(argv=None):
handler = logging.StreamHandler()
handler.setFormatter(default_formatter)
log.addHandler(handler)
logging.getLogger('torba').addHandler(handler)
logging.getLogger('torba').setLevel(logging.INFO)
logging.getLogger('aioupnp').setLevel(logging.WARNING)
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
+28 -17
View File
@@ -5,6 +5,7 @@ import logging
import math
import binascii
import typing
import socket
from hashlib import sha256
from types import SimpleNamespace
import base58
@@ -17,6 +18,7 @@ import lbrynet.schema
from lbrynet import utils
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.server import BlobServer
@@ -63,6 +65,14 @@ async def get_external_ip(): # used if upnp is disabled or non-functioning
pass
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
info = await loop.getaddrinfo(
url, 'https',
proto=socket.IPPROTO_TCP,
)
return info[0][4][0]
class DatabaseComponent(Component):
component_name = DATABASE_COMPONENT
@@ -349,7 +359,7 @@ class DHTComponent(Component):
async def get_status(self):
return {
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
}
@@ -366,8 +376,8 @@ class DHTComponent(Component):
async def start(self):
log.info("start the dht")
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
@@ -379,14 +389,13 @@ class DHTComponent(Component):
asyncio.get_event_loop(),
self.component_manager.peer_manager,
node_id=self.get_node_id(),
internal_udp_port=self.conf.udp_port,
internal_udp_port=self.conf.dht_node_port,
udp_port=self.external_udp_port,
external_ip=external_ip,
peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout
peer_port=self.external_peer_port
)
self.dht_node.start(
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
)
log.info("Started the dht")
@@ -410,7 +419,7 @@ class HashAnnouncerComponent(Component):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
self.hash_announcer.start(self.conf.concurrent_announcers)
log.info("Started blob announcer")
async def stop(self):
@@ -453,13 +462,17 @@ class StreamManagerComponent(Component):
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.stream_manager = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node,
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
self.conf.peer_connect_timeout, [
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
for url, port in self.conf.reflector_servers
], self.conf.reflector_servers
)
await self.stream_manager.start()
log.info('Done setting up file manager')
async def stop(self):
self.stream_manager.stop()
await self.stream_manager.stop()
class PeerProtocolServerComponent(Component):
@@ -479,10 +492,10 @@ class PeerProtocolServerComponent(Component):
upnp = self.component_manager.get_component(UPNP_COMPONENT)
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
peer_port = upnp.upnp_redirects.get("TCP", self.conf.tcp_port)
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
address = await wallet.get_unused_address()
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
self.blob_server.start_server(peer_port, interface='0.0.0.0')
await self.blob_server.started_listening.wait()
async def stop(self):
@@ -495,8 +508,8 @@ class UPnPComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = self.conf.tcp_port
self._int_dht_node_port = self.conf.udp_port
self._int_peer_port = self.conf.peer_port
self._int_dht_node_port = self.conf.dht_node_port
self.use_upnp = self.conf.use_upnp
self.upnp = None
self.upnp_redirects = {}
@@ -601,9 +614,7 @@ class UPnPComponent(Component):
else:
log.error("failed to setup upnp")
if self.component_manager.analytics_manager:
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
success, await self.get_status()
)
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
async def stop(self):
+108 -109
View File
@@ -6,10 +6,10 @@ import inspect
import typing
import aiohttp
import base58
import random
from urllib.parse import urlencode, quote
from typing import Callable, Optional, List
from binascii import hexlify, unhexlify
from copy import deepcopy
from traceback import format_exc
from aiohttp import web
from functools import wraps
@@ -18,9 +18,9 @@ from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
from lbrynet import __version__, utils
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.blob_exchange.downloader import download_blob
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
from lbrynet.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
from lbrynet.extras import system_info
from lbrynet.extras.daemon import analytics
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
@@ -29,7 +29,6 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
from lbrynet.extras.daemon.mime_types import guess_media_type
from lbrynet.extras.daemon.undecorated import undecorated
from lbrynet.extras.wallet.account import Account as LBCAccount
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
@@ -256,7 +255,6 @@ class Daemon(metaclass=JSONRPCServerType):
skip_components=conf.components_to_skip or []
)
self.component_startup_task = None
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
app = web.Application()
@@ -369,18 +367,6 @@ class Daemon(metaclass=JSONRPCServerType):
if not os.path.isdir(self.conf.download_dir):
os.makedirs(self.conf.download_dir)
async def update_connection_status(self):
connected = await utils.async_check_connection()
self._connection_status = (self.component_manager.loop.time(), connected)
async def get_connection_status(self) -> str:
if self._connection_status[0] + 300 > self.component_manager.loop.time():
if not self._connection_status[1]:
await self.update_connection_status()
else:
await self.update_connection_status()
return CONNECTION_STATUS_CONNECTED if self._connection_status[1] else CONNECTION_STATUS_NETWORK
async def start(self):
log.info("Starting LBRYNet Daemon")
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
@@ -678,8 +664,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
connection_code = await self.get_connection_status()
connection_code = CONNECTION_STATUS_NETWORK
response = {
'installation_id': self.installation_id,
'is_running': all(self.component_manager.get_components_status().values()),
@@ -1322,9 +1307,8 @@ class Daemon(metaclass=JSONRPCServerType):
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
[--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
[--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
[--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>]
[--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>]
[--comparison=<comparison>] [--full_status=<full_status>] [--reverse]
[--claim_name=<claim_name>] [--sort=<sort_by>] [--reverse] [--comparison=<comparison>]
[--full_status=<full_status>]
Options:
--sd_hash=<sd_hash> : (str) get file with matching sd hash
@@ -1337,12 +1321,11 @@ class Daemon(metaclass=JSONRPCServerType):
--txid=<txid> : (str) get file with matching claim txid
--nout=<nout> : (int) get file with matching claim nout
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
--channel_name=<channel_name> : (str) get file with matching channel name
--channel_name=<channel_name> : (str) get file with matching channel name
--claim_name=<claim_name> : (str) get file with matching claim name
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
--comparison=<comparison> : (str) logical comparision, (eq | ne | g | ge | l | le)
--sort=<sort_method> : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns:
(list) List of files
@@ -1361,24 +1344,21 @@ class Daemon(metaclass=JSONRPCServerType):
'download_path': (str) download path of file,
'mime_type': (str) mime type of file,
'key': (str) key attached to file,
'total_bytes_lower_bound': (int) lower bound file size in bytes,
'total_bytes': (int) file upper bound size in bytes,
'total_bytes': (int) file size in bytes,
'written_bytes': (int) written size in bytes,
'blobs_completed': (int) number of fully downloaded blobs,
'blobs_in_stream': (int) total blobs on stream,
'blobs_remaining': (int) total blobs remaining to download,
'status': (str) downloader status
'claim_id': (str) None if claim is not found else the claim id,
'outpoint': (str) None if claim is not found else the tx and output,
'txid': (str) None if claim is not found else the transaction id,
'nout': (int) None if claim is not found else the transaction output index,
'outpoint': (str) None if claim is not found else the tx and output,
'metadata': (dict) None if claim is not found else the claim metadata,
'channel_claim_id': (str) None if claim is not found or not signed,
'channel_name': (str) None if claim is not found or not signed,
'claim_name': (str) None if claim is not found else the claim name
},
]
}
"""
sort = sort or 'status'
comparison = comparison or 'eq'
@@ -1571,12 +1551,44 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
stream = await self.stream_manager.download_stream_from_uri(
uri, self.exchange_rate_manager, file_name, timeout
)
timeout = timeout if timeout is not None else self.conf.download_timeout
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise Exception("cannot download a channel claim, specify a /path")
resolved = (await self.wallet_manager.resolve(uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
)
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = ClaimDict.load_dict(resolved['value'])
fee_amount, fee_address = None, None
if claim.has_fee:
fee_amount = round(self.exchange_rate_manager.convert_currency(
claim.source_fee.currency, "LBC", claim.source_fee.amount
), 5)
fee_address = claim.source_fee.address
outpoint = f"{resolved['txid']}:{resolved['nout']}"
existing = self.stream_manager.get_filtered_streams(outpoint=outpoint)
if not existing:
existing.extend(self.stream_manager.get_filtered_streams(claim_id=resolved['claim_id'],
sd_hash=claim.source_hash))
if existing:
log.info("already have matching stream for %s", uri)
stream = existing[0]
else:
stream = await self.stream_manager.download_stream_from_claim(
self.dht_node, self.conf.download_dir, resolved, file_name, timeout, fee_amount, fee_address
)
if stream:
return stream.as_dict()
raise DownloadSDTimeout(uri)
raise DownloadSDTimeout(resolved['value']['stream']['source']['source'])
@requires(STREAM_MANAGER_COMPONENT)
async def jsonrpc_file_set_status(self, status, **kwargs):
@@ -1606,11 +1618,11 @@ class Daemon(metaclass=JSONRPCServerType):
if not streams:
raise Exception(f'Unable to find a file for {kwargs}')
stream = streams[0]
if status == 'start' and not stream.running:
await self.stream_manager.start_stream(stream)
if status == 'start' and not stream.running and not stream.finished:
stream.downloader.download(self.dht_node)
msg = "Resumed download"
elif status == 'stop' and stream.running:
stream.stop_download()
await stream.stop_download()
msg = "Stopped download"
else:
msg = (
@@ -1817,7 +1829,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def jsonrpc_publish(
self, name, bid, metadata=None, file_path=None, fee=None, title=None,
description=None, author=None, language=None, license=None,
license_url=None, thumbnail=None, preview=None, nsfw=None,
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
claim_address=None, change_address=None):
"""
@@ -1841,7 +1853,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
[--description=<description>] [--author=<author>] [--language=<language>]
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
[--preview=<preview>] [--nsfw=<nsfw>]
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
[--claim_address=<claim_address>] [--change_address=<change_address>]
@@ -1877,6 +1889,7 @@ class Daemon(metaclass=JSONRPCServerType):
--thumbnail=<thumbnail> : (str) thumbnail url
--preview=<preview> : (str) preview url
--nsfw=<nsfw> : (bool) whether the content is nsfw
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
@@ -1900,9 +1913,7 @@ class Daemon(metaclass=JSONRPCServerType):
"""
try:
parsed = parse_lbry_uri(name)
if parsed.name != name:
raise Exception("Name given to publish has invalid characters")
parse_lbry_uri(name)
except (TypeError, URIParseError):
raise Exception("Invalid name given to publish")
@@ -1918,10 +1929,9 @@ class Daemon(metaclass=JSONRPCServerType):
account = self.get_account_or_default(account_id)
available = await account.get_balance()
existing_claims = await account.get_claims(claim_name=name)
if len(existing_claims) > 1:
log.warning("found %i claims for the name %s", len(existing_claims), name)
existing_claims = []
if amount >= available:
existing_claims = await account.get_claims(claim_name=name)
if len(existing_claims) == 1:
available += existing_claims[0].get_estimator(self.ledger).effective_amount
if amount >= available:
@@ -1980,64 +1990,39 @@ class Daemon(metaclass=JSONRPCServerType):
}
}
sd_to_delete = None
if file_path:
# this will be used to verify the format with lbrynet.schema
claim_copy = deepcopy(claim_dict)
if sources is not None:
claim_dict['stream']['source'] = sources
claim_copy['stream']['source'] = sources
elif file_path is not None:
if not os.path.isfile(file_path):
raise Exception("invalid file path to publish")
if os.path.getsize(file_path) == 0:
raise Exception(f"Cannot publish empty file {file_path}")
if existing_claims:
sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source']
# since the file hasn't yet been made into a stream, we don't have
# a valid Source for the claim when validating the format, we'll use a fake one
claim_dict['stream']['source'] = {
claim_copy['stream']['source'] = {
'version': '_0_0_1',
'sourceType': 'lbry_sd_hash',
'source': '0' * 96,
'contentType': ''
}
elif not existing_claims:
raise Exception("no previous stream to update")
else:
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
# there is no existing source to use, and a file was not provided to make a new one
raise Exception("no source provided to publish")
try:
claim_dict = ClaimDict.load_dict(claim_dict).claim_dict
ClaimDict.load_dict(claim_copy)
# the metadata to use in the claim can be serialized by lbrynet.schema
except DecodeError as err:
# there was a problem with a metadata field, raise an error here rather than
# waiting to find out when we go to publish the claim (after having made the stream)
raise Exception(f"invalid publish metadata: {err}")
certificate = None
if channel_id or channel_name:
certificate = await self.get_channel_or_error(
self.get_accounts_or_all(channel_account_id), channel_id, channel_name
)
if file_path:
stream = await self.stream_manager.create_stream(file_path)
await self.storage.save_published_file(stream.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
claim_dict['stream']['source']['source'] = stream.sd_hash
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
if sd_to_delete:
stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete)
stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete)
if stream_to_delete:
await self.stream_manager.delete_stream(stream_to_delete, delete_file=False)
log.info("updating claim to stream generated from %s, deleted previous stream %s",
sd_to_delete[:8], file_path)
else:
log.info("previous stream %s from claim was not saved locally, nothing to delete",
sd_to_delete[:8])
else:
log.info("generated stream from %s for claim", file_path)
else:
log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8])
sd_hash = claim_dict['stream']['source']['source']
log.info("Publish: %s", {
'name': name,
'file_path': file_path,
@@ -2048,14 +2033,35 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel_id,
'channel_name': channel_name
})
from lbrynet.extras.daemon.mime_types import guess_media_type
if file_path:
if not os.path.isfile(file_path):
raise Exception(f"File {file_path} not found")
if os.path.getsize(file_path) == 0:
raise Exception(f"Cannot publish empty file {file_path}")
claim_dict['stream']['source'] = {}
stream = await self.stream_manager.create_stream(file_path)
stream_hash = stream.stream_hash
await self.storage.save_published_file(stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
claim_dict['stream']['source']['source'] = stream.sd_hash
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
else:
if not ('source' not in claim_dict['stream'] and existing_claims):
raise Exception("no previous stream to update")
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source'])
tx = await self.wallet_manager.claim_name(
account, name, amount, claim_dict, certificate, claim_address
)
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
if stream_hash:
await self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
await self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
await self.analytics_manager.send_claim_action('publish')
nout = 0
txo = tx.outputs[nout]
@@ -2283,7 +2289,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
claims = await self.wallet_manager.get_claims_for_name(name) # type: dict
claims['claims'] = sort_claim_results(claims['claims'])
sort_claim_results(claims['claims'])
return claims
@requires(WALLET_COMPONENT)
@@ -2563,7 +2569,8 @@ class Daemon(metaclass=JSONRPCServerType):
(str) Success/Fail message or (dict) decoded data
"""
blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash)
downloader = BlobDownloader(asyncio.get_event_loop(), self.blob_manager, self.conf)
blob = await downloader.get_blob(blob_hash, self.dht_node)
if read:
with open(blob.file_path, 'rb') as handle:
return handle.read().decode()
@@ -2784,19 +2791,7 @@ class Daemon(metaclass=JSONRPCServerType):
(list) list of blobs reflected
"""
server, port = kwargs.get('server'), kwargs.get('port')
if server and port:
port = int(port)
else:
server, port = random.choice(self.conf.reflector_servers)
reflected = await asyncio.gather(*[
stream.upload_to_reflector(server, port)
for stream in self.stream_manager.get_filtered_streams(**kwargs)
])
total = []
for reflected_for_stream in reflected:
total.extend(reflected_for_stream)
return total
raise NotImplementedError()
@requires(DHT_COMPONENT)
async def jsonrpc_peer_ping(self, node_id, address, port):
@@ -2812,14 +2807,18 @@ class Daemon(metaclass=JSONRPCServerType):
"""
peer = None
if node_id and address and port:
peer = self.component_manager.peer_manager.get_kademlia_peer(unhexlify(node_id), address,
udp_port=int(port))
try:
return await self.dht_node.protocol.get_rpc_peer(peer).ping()
except asyncio.TimeoutError:
return {'error': 'timeout'}
peer = self.component_manager.peer_manager.get_peer(address, unhexlify(node_id), udp_port=int(port))
if not peer:
peer = self.component_manager.peer_manager.make_peer(
address, unhexlify(node_id), udp_port=int(port)
)
if not peer:
return {'error': 'peer not found'}
try:
result = await peer.ping()
return result.decode()
except TimeoutError:
return {'error': 'ping timeout'}
@requires(DHT_COMPONENT)
def jsonrpc_routing_table_get(self):
+1 -4
View File
@@ -5,12 +5,9 @@ import logging
import aiohttp
from lbrynet import utils
from lbrynet.conf import Config
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
from lbrynet.extras import system_info
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
+11 -19
View File
@@ -3,7 +3,6 @@ import sqlite3
import typing
import asyncio
import binascii
import time
from torba.client.basedatabase import SQLiteMixin
from lbrynet.conf import Config
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
@@ -185,12 +184,11 @@ class SQLiteStorage(SQLiteMixin):
);
"""
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
def __init__(self, conf: Config, path, loop=None):
super().__init__(path)
self.conf = conf
self.content_claim_callbacks = {}
self.loop = loop or asyncio.get_event_loop()
self.time_getter = time_getter or time.time
async def run_and_return_one_or_none(self, query, *args):
for row in await self.db.execute_fetchall(query, args):
@@ -249,9 +247,8 @@ class SQLiteStorage(SQLiteMixin):
"select count(*) from blob where status='finished'"
)
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
def update_last_announced_blobs(self, blob_hashes: typing.List[str], last_announced: float):
def _update_last_announced_blobs(transaction: sqlite3.Connection):
last_announced = self.time_getter()
return transaction.executemany(
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
"where blob_hash=?",
@@ -262,7 +259,7 @@ class SQLiteStorage(SQLiteMixin):
def should_single_announce_blobs(self, blob_hashes, immediate=False):
def set_single_announce(transaction):
now = int(self.time_getter())
now = self.loop.time()
for blob_hash in blob_hashes:
if immediate:
transaction.execute(
@@ -277,8 +274,8 @@ class SQLiteStorage(SQLiteMixin):
def get_blobs_to_announce(self):
def get_and_update(transaction):
timestamp = int(self.time_getter())
if self.conf.announce_head_and_sd_only:
timestamp = self.loop.time()
if self.conf.announce_head_blobs_only:
r = transaction.execute(
"select blob_hash from blob "
"where blob_hash is not null and "
@@ -402,15 +399,16 @@ class SQLiteStorage(SQLiteMixin):
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
return self.save_published_file(
stream_hash, file_name, download_directory, data_payment_rate, status="running"
stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
status="running"
)
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
status="finished"):
return self.db.execute(
"insert into file values (?, ?, ?, ?, ?)",
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
(stream_hash, file_name, download_directory, data_payment_rate, status)
)
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
@@ -449,12 +447,6 @@ class SQLiteStorage(SQLiteMixin):
log.info("update file status %s -> %s", stream_hash, new_status)
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
stream_hash
))
def get_all_stream_hashes(self):
return self.run_and_return_list("select stream_hash from stream")
@@ -690,7 +682,7 @@ class SQLiteStorage(SQLiteMixin):
if success:
return self.db.execute(
"insert or replace into reflected_stream values (?, ?, ?)",
(sd_hash, reflector_address, time.time())
(sd_hash, reflector_address, self.loop.time())
)
return self.db.execute(
"delete from reflected_stream where sd_hash=? and reflector_address=?",
@@ -702,5 +694,5 @@ class SQLiteStorage(SQLiteMixin):
"select s.sd_hash from stream s "
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
"where r.timestamp is null or r.timestamp < ?",
int(self.time_getter()) - 86400
self.loop.time() - self.conf.auto_re_reflect_interval
)
+24 -31
View File
@@ -43,11 +43,11 @@ class StreamAssembler:
self.written_bytes: int = 0
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
if not blob or self.stream_handle.closed:
return False
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
def _decrypt_and_write():
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
if self.stream_handle.closed:
return False
self.stream_handle.seek(offset)
_decrypted = blob.decrypt(
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
@@ -55,54 +55,47 @@ class StreamAssembler:
self.stream_handle.write(_decrypted)
self.stream_handle.flush()
self.written_bytes += len(_decrypted)
log.debug("decrypted %s", blob.blob_hash[:8])
self.wrote_bytes_event.set()
return True
await self.loop.run_in_executor(None, _decrypt_and_write)
async def setup(self):
pass
async def after_got_descriptor(self):
pass
async def after_finished(self):
pass
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
if decrypted:
log.info("decrypted %s", blob.blob_hash[:8])
return
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
if not os.path.isdir(output_dir):
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
await self.setup()
self.sd_blob = await self.get_blob(self.sd_hash)
await self.blob_manager.blob_completed(self.sd_blob)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
self.sd_blob)
self.output_path = await get_next_available_file_name(self.loop, output_dir,
output_file_name or self.descriptor.suggested_file_name)
if not self.got_descriptor.is_set():
self.got_descriptor.set()
await self.after_got_descriptor()
self.output_path = await get_next_available_file_name(self.loop, output_dir,
output_file_name or self.descriptor.suggested_file_name)
self.stream_handle = open(self.output_path, 'wb')
await self.blob_manager.storage.store_stream(
self.sd_blob, self.descriptor
)
with open(self.output_path, 'wb') as stream_handle:
self.stream_handle = stream_handle
try:
for blob_info in self.descriptor.blobs[:-1]:
while True:
try:
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
break
except FileNotFoundError:
log.debug("stream assembler stopped")
return
except (ValueError, IOError, OSError):
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
self.descriptor.sd_hash)
except ValueError as err:
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
self.descriptor.sd_hash, str(err))
continue
self.stream_finished_event.set()
await self.after_finished()
if not self.wrote_bytes_event.is_set():
self.wrote_bytes_event.set()
self.stream_finished_event.set()
finally:
self.stream_handle.close()
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
return self.blob_manager.get_blob(blob_hash, length)
f = asyncio.Future(loop=self.loop)
f.set_result(self.blob_manager.get_blob(blob_hash, length))
return await f
+207 -61
View File
@@ -1,14 +1,13 @@
import os
import asyncio
import typing
import logging
from lbrynet.utils import resolve_host
from lbrynet.utils import drain_tasks, cancel_task
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
@@ -20,70 +19,217 @@ def drain_into(a: list, b: list):
b.append(a.pop())
class StreamDownloader(StreamAssembler):
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
output_file_name: typing.Optional[str] = None,
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
max_connections_per_stream: typing.Optional[int] = 8):
super().__init__(loop, blob_manager, sd_hash)
self.config = config
self.output_dir = output_dir or self.config.download_dir
self.peer_timeout = peer_timeout
self.peer_connect_timeout = peer_connect_timeout
self.current_blob: 'BlobFile' = None
self.download_task: asyncio.Task = None
self.accumulate_connections_task: asyncio.Task = None
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.output_dir = output_dir or os.getcwd()
self.output_file_name = output_file_name
self.blob_downloader: typing.Optional[BlobDownloader] = None
self.search_queue = asyncio.Queue(loop=loop)
self.peer_queue = asyncio.Queue(loop=loop)
self.accumulate_task: typing.Optional[asyncio.Task] = None
self.descriptor: typing.Optional[StreamDescriptor]
self.node: typing.Optional['Node'] = None
self.assemble_task: typing.Optional[asyncio.Task] = None
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
self._lock = asyncio.Lock(loop=self.loop)
self.max_connections_per_stream = max_connections_per_stream
self.fixed_peers = fixed_peers or []
async def setup(self): # start the peer accumulator and initialize the downloader
if self.blob_downloader:
raise Exception("downloader is already set up")
if self.node:
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.search_queue.put_nowait(self.sd_hash)
async def _update_current_blob(self, blob: 'BlobFile'):
async with self._lock:
drain_tasks(self.running_download_requests)
self.current_blob = blob
if not blob.get_is_verified():
self._update_requests()
async def after_got_descriptor(self):
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to search")
async def _request_blob(self, peer: 'KademliaPeer'):
if self.current_blob.get_is_verified():
log.info("already verified")
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
peer.address, peer.tcp_port, self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self._lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
async def after_finished(self):
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
def _update_requests(self):
self.new_peer_event.clear()
if self.current_blob.blob_hash not in self.requested_from:
self.requested_from[self.current_blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.current_blob.blob_hash][peer] = task
self.running_download_requests.append(task)
def stop(self):
if self.accumulate_task:
self.accumulate_task.cancel()
self.accumulate_task = None
if self.assemble_task:
self.assemble_task.cancel()
self.assemble_task = None
if self.fixed_peers_handle:
self.fixed_peers_handle.cancel()
self.fixed_peers_handle = None
self.blob_downloader = None
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
async with self._lock:
if len(self.running_download_requests) < self.max_connections_per_stream:
# update the running download requests
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
async with self._lock:
if self.current_blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.current_blob
else:
for task in download_tasks:
if task and not task.done():
self.running_download_requests.append(task)
return
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
return await self.blob_downloader.download_blob(blob_hash, length)
blob = self.blob_manager.get_blob(blob_hash, length)
await self._update_current_blob(blob)
if blob.get_is_verified():
return blob
def add_fixed_peers(self):
async def _add_fixed_peers():
self.peer_queue.put_nowait([
KademliaPeer(self.loop, address=(await resolve_host(url)), tcp_port=port + 1)
for url, port in self.config.reflector_servers
])
if self.config.reflector_servers:
self.fixed_peers_handle = self.loop.call_later(
self.config.fixed_peer_delay if (
'dht' not in self.config.components_to_skip
and self.node
and len(self.node.protocol.routing_table.get_peers())
) else 0.0,
lambda: self.loop.create_task(_add_fixed_peers())
# the blob must be downloaded
try:
while not self.current_blob.get_is_verified():
if not self.active_connections: # wait for a new connection
await self.new_peer_event.wait()
continue
blob = await self.wait_for_download_or_new_peer()
if blob:
drain_tasks(self.running_download_requests)
return blob
return blob
except asyncio.CancelledError:
drain_tasks(self.running_download_requests)
raise
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
async def _accumulate_connections(self, node: 'Node'):
blob_queue = asyncio.Queue(loop=self.loop)
blob_queue.put_nowait(self.sd_hash)
task = asyncio.create_task(self.got_descriptor.wait())
added_peers = asyncio.Event(loop=self.loop)
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
if self.fixed_peers:
def check_added_peers():
if not added_peers.is_set():
self._add_peer_protocols(self.fixed_peers)
log.info("no dht peers for download yet, adding fixed peer")
added_peers.set()
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
def got_descriptor(f):
try:
f.result()
except asyncio.CancelledError:
return
log.info("add head blob hash to peer search")
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
task.add_done_callback(got_descriptor)
try:
async with node.stream_peer_search_junction(blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
if not added_peers.is_set():
added_peers.set()
return
except asyncio.CancelledError:
pass
finally:
if task and not task.done():
task.cancel()
log.info("cancelled head blob task")
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
add_fixed_peers_timer.cancel()
async def stop(self):
cancel_task(self.accumulate_connections_task)
self.accumulate_connections_task = None
drain_tasks(self.running_download_requests)
while self.requested_from:
_, peer_task_dict = self.requested_from.popitem()
while peer_task_dict:
peer, task = peer_task_dict.popitem()
try:
cancel_task(task)
except asyncio.CancelledError:
pass
while self.active_connections:
_, client = self.active_connections.popitem()
if client:
await client.close()
log.info("stopped downloader")
async def _download(self):
try:
log.info("download and decrypt stream")
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
log.info(
"downloaded stream %s -> %s", self.sd_hash, self.output_path
)
await self.blob_manager.storage.change_file_status(
self.descriptor.stream_hash, 'finished'
)
except asyncio.CancelledError:
pass
finally:
await self.stop()
def download(self, node: typing.Optional['Node'] = None):
self.node = node
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
self.add_fixed_peers()
def download(self, node: 'Node'):
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
self.download_task = self.loop.create_task(self._download())
+7 -32
View File
@@ -2,16 +2,13 @@ import os
import asyncio
import typing
import logging
import binascii
from lbrynet.extras.daemon.mime_types import guess_media_type
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.reflector.client import StreamReflectorClient
from lbrynet.extras.daemon.storage import StoredStreamClaim
if typing.TYPE_CHECKING:
from lbrynet.schema.claim import ClaimDict
from lbrynet.extras.daemon.storage import StoredStreamClaim
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.dht.node import Node
log = logging.getLogger(__name__)
@@ -23,7 +20,7 @@ class ManagedStream:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None):
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
self.loop = loop
self.blob_manager = blob_manager
self.download_directory = download_directory
@@ -33,6 +30,7 @@ class ManagedStream:
self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim
self._status = status
self._store_after_finished: asyncio.Task = None
self.fully_reflected = asyncio.Event(loop=self.loop)
@property
@@ -100,17 +98,13 @@ class ManagedStream:
def sd_hash(self):
return self.descriptor.sd_hash
@property
def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed
def as_dict(self) -> typing.Dict:
full_path = os.path.join(self.download_directory, self.file_name)
if not os.path.isfile(full_path):
if not os.path.exists(full_path):
full_path = None
mime_type = guess_media_type(os.path.basename(self.file_name))
if self.downloader and self.downloader.written_bytes:
if self.downloader:
written_bytes = self.downloader.written_bytes
elif full_path:
written_bytes = os.stat(full_path).st_size
@@ -134,7 +128,6 @@ class ManagedStream:
'written_bytes': written_bytes,
'blobs_completed': self.blobs_completed,
'blobs_in_stream': self.blobs_in_stream,
'blobs_remaining': self.blobs_remaining,
'status': self.status,
'claim_id': self.claim_id,
'txid': self.txid,
@@ -161,13 +154,9 @@ class ManagedStream:
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
status=cls.STATUS_FINISHED)
def start_download(self, node: typing.Optional['Node']):
self.downloader.download(node)
self.update_status(self.STATUS_RUNNING)
def stop_download(self):
async def stop_download(self):
if self.downloader:
self.downloader.stop()
await self.downloader.stop()
if not self.finished:
self.update_status(self.STATUS_STOPPED)
@@ -188,11 +177,6 @@ class ManagedStream:
sent_sd, needed = await protocol.send_descriptor()
if sent_sd:
sent.append(self.sd_hash)
if not sent_sd and not needed:
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return []
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
@@ -209,13 +193,4 @@ class ManagedStream:
protocol.transport.close()
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return sent
def set_claim(self, claim_info: typing.Dict, claim: 'ClaimDict'):
self.stream_claim_info = StoredStreamClaim(
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.serialized).decode(), claim.certificate_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
+2 -3
View File
@@ -15,7 +15,6 @@ log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.blob_manager = blob_manager
self.descriptor = descriptor
@@ -27,7 +26,7 @@ class StreamReflectorClient(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
log.debug("Connected to reflector")
log.info("Connected to reflector")
self.connected.set()
def connection_lost(self, exc: typing.Optional[Exception]):
@@ -46,7 +45,7 @@ class StreamReflectorClient(asyncio.Protocol):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.pending_request = self.loop.create_task(self.response_queue.get())
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
return await self.pending_request
finally:
self.pending_request = None
+3 -9
View File
@@ -37,8 +37,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
try:
self.writer.write(data)
except IOError as err:
log.error("error receiving blob: %s", err)
self.transport.close()
log.error("error downloading blob: %s", err)
return
try:
request = json.loads(data.decode())
@@ -68,33 +67,28 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.send_response({"send_sd_blob": True})
try:
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_sd_blob": True})
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
self.writer.close_handle()
self.writer = None
self.send_response({"received_sd_blob": True})
except (asyncio.TimeoutError, asyncio.CancelledError):
self.send_response({"received_sd_blob": False})
self.incoming.clear()
self.writer.close_handle()
self.writer = None
self.transport.close()
self.send_response({"received_sd_blob": False})
return
else:
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
if self.writer:
self.writer.close_handle()
self.writer = None
self.send_response({"send_sd_blob": False, 'needed': [
blob.blob_hash for blob in self.descriptor.blobs[:-1]
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
]})
return
return
elif self.descriptor:
if 'blob_hash' not in request:
+86 -203
View File
@@ -4,20 +4,16 @@ import typing
import binascii
import logging
import random
from lbrynet.error import ResolveError
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.decode import smart_decode
from lbrynet.extras.daemon.storage import lbc_to_dewies
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
@@ -35,9 +31,7 @@ filter_fields = [
'nout',
'channel_claim_id',
'channel_name',
'full_status', # TODO: remove
'blobs_remaining',
'blobs_in_stream'
'full_status'
]
comparison_operators = {
@@ -51,135 +45,81 @@ comparison_operators = {
class StreamManager:
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node']):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.wallet = wallet
self.storage = storage
self.node = node
self.peer_timeout = peer_timeout
self.peer_connect_timeout = peer_connect_timeout
self.streams: typing.Set[ManagedStream] = set()
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
self.resume_downloading_task: asyncio.Task = None
self.re_reflect_task: asyncio.Task = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash)
stream.set_claim(claim_info, smart_decode(claim_info['value']))
async def start_stream(self, stream: ManagedStream) -> bool:
"""
Resume or rebuild a partial or completed stream
"""
path = os.path.join(stream.download_directory, stream.file_name)
if not stream.running and not os.path.isfile(path):
if stream.downloader:
stream.downloader.stop()
stream.downloader = None
# the directory is gone, can happen when the folder that contains a published file is deleted
# reset the download directory to the default and update the file name
if not os.path.isdir(stream.download_directory):
stream.download_directory = self.config.download_dir
stream.downloader = self.make_downloader(
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
)
if stream.status != ManagedStream.STATUS_FINISHED:
await self.storage.change_file_status(stream.stream_hash, 'running')
stream.update_status('running')
stream.start_download(self.node)
try:
await asyncio.wait_for(self.loop.create_task(stream.downloader.got_descriptor.wait()),
self.config.download_timeout)
except asyncio.TimeoutError:
stream.stop_download()
stream.downloader = None
return False
file_name = os.path.basename(stream.downloader.output_path)
await self.storage.change_file_download_dir_and_file_name(
stream.stream_hash, self.config.download_dir, file_name
)
self.wait_for_stream_finished(stream)
return True
return True
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
return StreamDownloader(
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
)
async def add_stream(self, sd_hash: str, file_name: str, download_directory: str, status: str, claim):
sd_blob = self.blob_manager.get_blob(sd_hash)
if sd_blob.get_is_verified():
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
stream = ManagedStream(
self.loop, self.blob_manager, descriptor,
download_directory,
file_name,
downloader, status, claim
)
self.streams.add(stream)
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
self.fixed_peers = fixed_peers
self.reflector_servers = reflector_servers
async def load_streams_from_database(self):
file_infos = await self.storage.get_all_lbry_files()
await asyncio.gather(*[
self.add_stream(
file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'], file_info['claim']
) for file_info in file_infos
])
log.info("Started stream manager with %i files", len(file_infos))
infos = await self.storage.get_all_lbry_files()
for file_info in infos:
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
if sd_blob.get_is_verified():
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
downloader = StreamDownloader(
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
)
stream = ManagedStream(
self.loop, self.blob_manager, descriptor,
binascii.unhexlify(file_info['download_directory']).decode(),
binascii.unhexlify(file_info['file_name']).decode(),
downloader, file_info['status'], file_info['claim']
)
self.streams.add(stream)
async def resume(self):
if not self.node:
log.warning("no DHT node given, cannot resume downloads")
return
await self.node.joined.wait()
t = [
stream.start_download(self.node)
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
]
if t:
log.info("resuming %i downloads", len(t))
resumed = 0
for stream in self.streams:
if stream.status == ManagedStream.STATUS_RUNNING:
resumed += 1
stream.downloader.download(self.node)
self.wait_for_stream_finished(stream)
if resumed:
log.info("resuming %i downloads", resumed)
async def reflect_streams(self):
while True:
if self.config.reflector_servers:
sd_hashes = await self.storage.get_streams_to_re_reflect()
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
streams = list(self.streams)
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= 10:
await asyncio.gather(*batch)
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.config.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= self.config.concurrent_reflector_uploads:
await asyncio.gather(*batch)
batch = []
if batch:
await asyncio.gather(*batch)
await asyncio.sleep(300, loop=self.loop)
if batch:
await asyncio.gather(*batch)
async def start(self):
await self.load_streams_from_database()
self.resume_downloading_task = self.loop.create_task(self.resume())
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
def stop(self):
async def stop(self):
if self.resume_downloading_task and not self.resume_downloading_task.done():
self.resume_downloading_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel()
while self.streams:
stream = self.streams.pop()
stream.stop_download()
await stream.stop_download()
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
@@ -187,25 +127,24 @@ class StreamManager:
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
self.streams.add(stream)
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflector_servers:
host, port = random.choice(self.config.reflector_servers)
if self.reflector_servers:
host, port = random.choice(self.reflector_servers)
self.loop.create_task(stream.upload_to_reflector(host, port))
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
stream_finished = False if not stream.finished and stream.downloader\
else (stream.downloader and stream.downloader.stream_finished_event.is_set())
if not stream_finished:
delete_file = True
stream.stop_download()
await stream.stop_download()
self.streams.remove(stream)
await self.storage.delete_stream(stream.descriptor)
blob_hashes = [stream.sd_hash]
for blob_info in stream.descriptor.blobs[:-1]:
blob_hashes.append(blob_info.blob_hash)
await self.blob_manager.delete_blobs(blob_hashes)
for blob_hash in blob_hashes:
blob = self.blob_manager.get_blob(blob_hash)
if blob.get_is_verified():
await blob.delete()
if delete_file:
path = os.path.join(stream.download_directory, stream.file_name)
if os.path.isfile(path):
@@ -227,70 +166,76 @@ class StreamManager:
)
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
sd_blob_timeout: typing.Optional[float] = 60
) -> typing.Optional[ManagedStream]:
claim = smart_decode(claim_info['value'])
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
download_directory, file_name)
claim = ClaimDict.load_dict(claim_info['value'])
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
try:
downloader.download(node)
await downloader.got_descriptor.wait()
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
except (asyncio.TimeoutError, asyncio.CancelledError):
log.info("stream timeout")
downloader.stop()
await downloader.stop()
log.info("stopped stream")
return
file_name = os.path.basename(downloader.output_path)
download_directory = os.path.dirname(downloader.output_path)
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
await self.blob_manager.storage.save_downloaded_file(
downloader.descriptor.stream_hash, file_name, download_directory,
0.0
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
data_rate
)
await self.blob_manager.storage.save_content_claim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
)
stored_claim = StoredStreamClaim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
claim_info.get('channel_name')
)
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
file_name, downloader, ManagedStream.STATUS_RUNNING)
stream.set_claim(claim_info, claim)
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
stored_claim)
self.streams.add(stream)
try:
await stream.downloader.wrote_bytes_event.wait()
self.wait_for_stream_finished(stream)
return stream
except asyncio.CancelledError:
downloader.stop()
log.debug("stopped stream")
await downloader.stop()
async def download_stream_from_claim(self, node: 'Node', claim_info: typing.Dict,
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = 60,
sd_blob_timeout: typing.Optional[float] = 60,
fee_amount: typing.Optional[float] = 0.0,
fee_address: typing.Optional[str] = None,
should_pay: typing.Optional[bool] = True) -> typing.Optional[ManagedStream]:
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
claim = ClaimDict.load_dict(claim_info['value'])
if fee_address and fee_amount:
if fee_amount > await self.wallet.default_account.get_balance():
raise Exception("not enough funds")
sd_hash = claim.source_hash.decode()
if sd_hash in self.starting_streams:
return await self.starting_streams[sd_hash]
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
if already_started:
return already_started[0]
if should_pay and fee_address and fee_amount and fee_amount > await self.wallet.default_account.get_balance():
raise Exception("not enough funds")
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
stream_task = self.loop.create_task(
self._download_stream_from_claim(node, self.config.download_dir, claim_info, file_name)
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
)
try:
await asyncio.wait_for(stream_task, timeout or self.config.download_timeout)
await asyncio.wait_for(stream_task, sd_blob_timeout)
stream = await stream_task
self.starting_streams[sd_hash].set_result(stream)
if should_pay and fee_address and fee_amount:
if fee_address and fee_amount:
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
return stream
except (asyncio.TimeoutError, asyncio.CancelledError):
@@ -300,11 +245,6 @@ class StreamManager:
del self.starting_streams[sd_hash]
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
if streams:
return streams[0]
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
comparison: typing.Optional[str] = None,
**search_by) -> typing.List[ManagedStream]:
@@ -340,60 +280,3 @@ class StreamManager:
if reverse:
streams.reverse()
return streams
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> typing.Optional[ManagedStream]:
timeout = timeout or self.config.download_timeout
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise Exception("cannot download a channel claim, specify a /path")
resolved = (await self.wallet.resolve(uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
)
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = ClaimDict.load_dict(resolved['value'])
fee_amount, fee_address = None, None
if claim.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.source_fee.currency, "LBC", claim.source_fee.amount
), 5)
fee_address = claim.source_fee.address.decode()
outpoint = f"{resolved['txid']}:{resolved['nout']}"
existing = self.get_filtered_streams(outpoint=outpoint)
if not existing:
existing.extend(self.get_filtered_streams(sd_hash=claim.source_hash.decode()))
if existing and existing[0].claim_id != resolved['claim_id']:
raise Exception(f"stream for {existing[0].claim_id} collides with existing "
f"download {resolved['claim_id']}")
elif not existing:
existing.extend(self.get_filtered_streams(claim_id=resolved['claim_id']))
if existing and existing[0].sd_hash != claim.source_hash.decode():
log.info("claim contains an update to a stream we have, downloading it")
stream = await self.download_stream_from_claim(
self.node, resolved, file_name, timeout, fee_amount, fee_address, False
)
log.info("started new stream, deleting old one")
await self.delete_stream(existing[0])
return stream
elif existing:
log.info("already have matching stream for %s", uri)
stream = existing[0]
await self.start_stream(stream)
return stream
else:
stream = existing[0]
await self.start_stream(stream)
return stream
log.info("download stream from %s", uri)
return await self.download_stream_from_claim(
self.node, resolved, file_name, timeout, fee_amount, fee_address
)
+7 -24
View File
@@ -8,7 +8,6 @@ import json
import typing
import asyncio
import logging
import ipaddress
import pkg_resources
from lbrynet.schema.claim import ClaimDict
from lbrynet.cryptoutils import get_lbry_hash_obj
@@ -69,12 +68,13 @@ def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
def check_connection(server="lbry.io", port=80, timeout=5):
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
server = socket.gethostbyname(server)
socket.create_connection((server, port), timeout).close()
conn = socket.create_connection((server, port), timeout)
conn.close()
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
@@ -83,23 +83,19 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
try:
server = "8.8.8.8"
port = 53
socket.create_connection((server, port), timeout).close()
socket.create_connection((server, port), timeout)
log.debug('Connection successful')
return True
except Exception:
except Exception as ex:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
except Exception:
except Exception as ex:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
server, port)
return False
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)])
@@ -140,16 +136,3 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
while tasks:
cancel_task(tasks.pop())
async def resolve_host(url: str) -> str:
try:
if ipaddress.ip_address(url):
return url
except ValueError:
pass
loop = asyncio.get_running_loop()
return (await loop.getaddrinfo(
url, 'https',
proto=socket.IPPROTO_TCP,
))[0][4][0]
-53
View File
@@ -1,53 +0,0 @@
import sys
import os
import asyncio
import socket
import ipaddress
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
import logging
log = logging.getLogger("lbrynet")
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
async def main(blob_hash: str, url: str):
conf = Config()
loop = asyncio.get_running_loop()
host_url, port = url.split(":")
try:
host = None
if ipaddress.ip_address(host_url):
host = host_url
except ValueError:
host = None
if not host:
host_info = await loop.getaddrinfo(
host_url, 'https',
proto=socket.IPPROTO_TCP,
)
host = host_info[0][4][0]
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
await storage.open()
await blob_manager.setup()
blob = blob_manager.get_blob(blob_hash)
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
conf.blob_download_timeout)
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
f"keep connection: {keep}")
if blob.get_is_verified():
await blob_manager.delete_blobs([blob.blob_hash])
print(f"deleted {blob_hash}")
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
url = 'reflector.lbry.io:5567'
if len(sys.argv) > 2:
url = sys.argv[2]
asyncio.run(main(sys.argv[1], url))
-33
View File
@@ -1,33 +0,0 @@
import os
import binascii
import sqlite3
from lbrynet.conf import Config
def main():
conf = Config()
db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite'))
cur = db.cursor()
files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall()
update = {}
for stream_hash, file_name, download_directory in files:
try:
binascii.unhexlify(file_name)
except binascii.Error:
try:
binascii.unhexlify(download_directory)
except binascii.Error:
update[stream_hash] = (
binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode()
)
if update:
print(f"repair {len(update)} streams")
for stream_hash, (file_name, download_directory) in update.items():
cur.execute('update file set file_name=?, download_directory=? where stream_hash=?',
(file_name, download_directory, stream_hash))
db.commit()
db.close()
if __name__ == "__main__":
main()
+11 -8
View File
@@ -57,29 +57,31 @@ def variance(times):
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(conf, uri):
async def wait_for_done(api, uri):
name = uri.split("#")[0]
last_complete = 0
hang_count = 0
while True:
files = await daemon_rpc(conf, "file_list", claim_name=name)
files = await api.file_list(claim_name=name)
file = files[0]
if file['status'] in ['finished', 'stopped']:
return True, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
if last_complete < int(file['blobs_completed']):
print(f"{file['blobs_completed']}/{file['blobs_in_stream']}...")
hang_count = 0
last_complete = int(file['blobs_completed'])
else:
hang_count += 1
await asyncio.sleep(1.0)
if hang_count > 10:
if hang_count > 30:
return False, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
async def main(uris=None):
async def main(start_daemon=True, uris=None):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
daemon = None
try:
await daemon_rpc(conf, 'status')
except (ClientConnectorError, ConnectionError):
@@ -111,12 +113,12 @@ async def main(uris=None):
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
# downloaded, msg, blobs_in_stream = await wait_for_done(conf, uri)
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
# if downloaded:
# downloaded_times.append((time.time()-start) / downloaded)
# print(f"\tdownloaded {uri} @ {(time.time()-start) / blobs_in_stream} seconds per blob")
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
# else:
# print(f"\tfailed to download {uri}, got {msg}")
# print(f"failed to downlload {uri}, got {msg}")
# download_failures.append(uri)
except:
print(f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}")
@@ -134,7 +136,8 @@ async def main(uris=None):
nt = '\n\t'
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
print(result)
if daemon:
await daemon.shutdown()
# webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
# if webhook:
# await report_to_slack(result, webhook)
-1
View File
@@ -37,6 +37,5 @@ setup(
'torba',
'pyyaml',
'docopt',
'colorama==0.3.7',
],
)
+1 -2
View File
@@ -41,10 +41,9 @@ class CommandTestCase(IntegrationTestCase):
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_streams = False
conf.reflect_uploads = False
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('localhost', 50001)]
conf.reflector_servers = []
conf.known_dht_nodes = []
await self.account.ensure_address_gap()
+16 -36
View File
@@ -1,16 +1,12 @@
import asyncio
import tempfile
from io import BytesIO
import shutil
import os
from lbrynet.blob_exchange.serialization import BlobRequest
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
from lbrynet.dht.peer import KademliaPeer, PeerManager
@@ -18,12 +14,6 @@ from lbrynet.dht.peer import KademliaPeer, PeerManager
# logging.getLogger("lbrynet").setLevel(logging.DEBUG)
def mock_config():
config = Config()
config.fixed_peer_delay = 10000
return config
class BlobExchangeTestBase(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
@@ -32,15 +22,12 @@ class BlobExchangeTestBase(AsyncioTestCase):
self.server_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.client_dir)
self.addCleanup(shutil.rmtree, self.server_dir)
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
reflector_servers=[])
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_storage = SQLiteStorage(Config(), os.path.join(self.server_dir, "lbrynet.sqlite"))
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
reflector_servers=[])
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_storage = SQLiteStorage(Config(), os.path.join(self.client_dir, "lbrynet.sqlite"))
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
self.client_peer_manager = PeerManager(self.loop)
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
@@ -66,10 +53,12 @@ class TestBlobExchange(BlobExchangeTestBase):
async def _test_transfer_blob(self, blob_hash: str):
client_blob = self.client_blob_manager.get_blob(blob_hash)
protocol = BlobExchangeClientProtocol(self.loop, 2)
# download the blob
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
self.server_from_client.tcp_port, 2, 3)
downloaded = await request_blob(self.loop, client_blob, protocol, self.server_from_client.address,
self.server_from_client.tcp_port, 2)
await protocol.close()
self.assertEqual(client_blob.get_is_verified(), True)
self.assertTrue(downloaded)
@@ -102,15 +91,17 @@ class TestBlobExchange(BlobExchangeTestBase):
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
protocol = BlobExchangeClientProtocol(self.loop, 2)
# download the blob
await asyncio.gather(
request_blob(
self.loop, second_client_blob, server_from_second_client.address,
server_from_second_client.tcp_port, 2, 3
self.loop, second_client_blob, protocol, server_from_second_client.address,
server_from_second_client.tcp_port, 2
),
self._test_transfer_blob(blob_hash)
)
await protocol.close()
self.assertEqual(second_client_blob.get_is_verified(), True)
async def test_host_different_blobs_to_multiple_peers_at_once(self):
@@ -134,25 +125,14 @@ class TestBlobExchange(BlobExchangeTestBase):
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
protocol = BlobExchangeClientProtocol(self.loop, 2)
await asyncio.gather(
request_blob(
self.loop, second_client_blob, server_from_second_client.address,
server_from_second_client.tcp_port, 2, 3
self.loop, second_client_blob, protocol, server_from_second_client.address,
server_from_second_client.tcp_port, 2
),
self._test_transfer_blob(sd_hash)
)
await protocol.close()
self.assertEqual(second_client_blob.get_is_verified(), True)
async def test_server_chunked_request(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address)
transport = asyncio.Transport(extra={'peername': ('ip', 90)})
received_data = BytesIO()
transport.write = received_data.write
server_protocol.connection_made(transport)
blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize()
for byte in blob_request:
server_protocol.data_received(bytes([byte]))
await asyncio.sleep(0.1) # yield execution
self.assertTrue(len(received_data.getvalue()) > 0)
+4 -7
View File
@@ -1,8 +1,6 @@
import asyncio
import binascii
from torba.testcase import AsyncioTestCase
from tests import dht_mocks
from lbrynet.dht.serialization.bencoding import bencode, bdecode
from lbrynet.dht import constants
from lbrynet.dht.protocol.protocol import KademliaProtocol
from lbrynet.dht.peer import PeerManager
@@ -69,7 +67,6 @@ class TestProtocol(AsyncioTestCase):
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
)
peer2_from_peer1.update_tcp_port(3333)
peer3 = peer1.peer_manager.get_kademlia_peer(
constants.generate_id(), '1.2.3.6', udp_port=4444
)
@@ -79,14 +76,14 @@ class TestProtocol(AsyncioTestCase):
self.assertEqual(True, peer1.data_store.has_peers_for_blob(b'2' * 48))
self.assertEqual(False, peer1.data_store.has_peers_for_blob(b'3' * 48))
self.assertListEqual([peer2_from_peer1], peer1.data_store.get_storing_contacts())
peer1.data_store.completed_blobs.add(binascii.hexlify(b'2' * 48).decode())
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
self.assertSetEqual(
{b'2' * 48, b'token', b'protocolVersion'}, set(find_value_response.keys())
)
self.assertEqual(2, len(find_value_response[b'2' * 48]))
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
self.assertEqual(1, len(find_value_response[b'2' * 48]))
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1)
# self.assertEqual(peer2_from_peer1.tcp_port, 3333)
peer1.stop()
peer2.stop()
+9 -11
View File
@@ -1,4 +1,3 @@
import unittest
import asyncio
from torba.testcase import AsyncioTestCase
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
@@ -42,17 +41,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
async def test_yield_order(self):
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2]
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
await self._test_junction(expected_order, fast_gen, slow_gen)
self.assertEqual(fast_gen.called_close, True)
self.assertEqual(slow_gen.called_close, True)
@unittest.SkipTest
async def test_one_stopped_first(self):
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
fast_gen = MockAsyncGen(self.loop, 1, 0.2, 5)
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
fast_gen = MockAsyncGen(self.loop, 1, 0.101, 5)
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
await self._test_junction(expected_order, fast_gen, slow_gen)
self.assertEqual(fast_gen.called_close, True)
self.assertEqual(slow_gen.called_close, True)
@@ -64,16 +62,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
for i in range(10):
if i == 5:
return
await asyncio.sleep(0.2)
await asyncio.sleep(0.101)
yield 1
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
await self._test_junction(expected_order, fast_gen(), slow_gen)
self.assertEqual(slow_gen.called_close, True)
async def test_stop_when_encapsulating_task_cancelled(self):
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
async def _task():
async with AsyncGeneratorJunction(self.loop) as junction:
@@ -83,7 +81,7 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
pass
task = self.loop.create_task(_task())
self.loop.call_later(1.0, task.cancel)
self.loop.call_later(0.5, task.cancel)
with self.assertRaises(asyncio.CancelledError):
await task
self.assertEqual(fast_gen.called_close, True)
+17
View File
@@ -84,6 +84,15 @@ class TestCostEst(unittest.TestCase):
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, correct_result)
def test_fee_and_ungenerous_data(self):
conf = Config(is_generous_host=False)
size = 10000000
fake_fee_amount = 4.5
correct_result = size / 10 ** 6 * conf.data_rate + fake_fee_amount
daemon = get_test_daemon(conf, with_fee=True)
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, round(correct_result, 1))
def test_generous_data_and_no_fee(self):
size = 10000000
correct_result = 0.0
@@ -91,6 +100,14 @@ class TestCostEst(unittest.TestCase):
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, correct_result)
def test_ungenerous_data_and_no_fee(self):
conf = Config(is_generous_host=False)
size = 10000000
correct_result = size / 10 ** 6 * conf.data_rate
daemon = get_test_daemon(conf)
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, round(correct_result, 1))
@unittest.SkipTest
class TestJsonRpc(unittest.TestCase):
+1 -1
View File
@@ -7,8 +7,8 @@ from lbrynet.conf import Config
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.assembler import StreamAssembler
class TestStreamAssembler(AsyncioTestCase):
+26 -26
View File
@@ -1,8 +1,7 @@
import os
import unittest
from unittest import mock
import asyncio
from lbrynet.conf import Config
import contextlib
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.dht.node import Node
@@ -22,58 +21,59 @@ class TestStreamDownloader(BlobExchangeTestBase):
f.write(self.stream_bytes)
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
self.sd_hash = descriptor.calculate_sd_hash()
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
reflector_servers=[])
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
self.downloader = StreamDownloader(self.loop, self.client_blob_manager, self.sd_hash, 3, 3, self.client_dir)
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
async def _test_transfer_stream(self, blob_count: int, mock_peer_search=None):
await self.setup_stream(blob_count)
mock_node = mock.Mock(spec=Node)
def _mock_accumulate_peers(q1, q2):
async def _task():
pass
q2.put_nowait([self.server_from_client])
return q2, self.loop.create_task(_task())
@contextlib.asynccontextmanager
async def _mock_peer_search(*_):
async def _gen():
yield [self.server_from_client]
return
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
yield _gen()
mock_node.stream_peer_search_junction = mock_peer_search or _mock_peer_search
self.downloader.download(mock_node)
await self.downloader.stream_finished_event.wait()
self.downloader.stop()
await self.downloader.stop()
self.assertTrue(os.path.isfile(self.downloader.output_path))
with open(self.downloader.output_path, 'rb') as f:
self.assertEqual(f.read(), self.stream_bytes)
await asyncio.sleep(0.01)
self.assertTrue(self.downloader.stream_handle.closed)
async def test_transfer_stream(self):
await self._test_transfer_stream(10)
@unittest.SkipTest
async def test_transfer_hundred_blob_stream(self):
await self._test_transfer_stream(100)
# async def test_transfer_hundred_blob_stream(self):
# await self._test_transfer_stream(100)
async def test_transfer_stream_bad_first_peer_good_second(self):
await self.setup_stream(2)
mock_node = mock.Mock(spec=Node)
q = asyncio.Queue()
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
def _mock_accumulate_peers(q1, q2):
async def _task():
pass
@contextlib.asynccontextmanager
async def mock_peer_search(*_):
async def _gen():
await asyncio.sleep(0.05, loop=self.loop)
yield [bad_peer]
await asyncio.sleep(0.1, loop=self.loop)
yield [self.server_from_client]
return
q2.put_nowait([bad_peer])
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
return q2, self.loop.create_task(_task())
yield _gen()
mock_node.accumulate_peers = _mock_accumulate_peers
mock_node.stream_peer_search_junction = mock_peer_search
self.downloader.download(mock_node)
await self.downloader.stream_finished_event.wait()
await self.downloader.stop()
self.assertTrue(os.path.isfile(self.downloader.output_path))
with open(self.downloader.output_path, 'rb') as f:
self.assertEqual(f.read(), self.stream_bytes)
+1 -1
View File
@@ -21,7 +21,7 @@ class TestStreamAssembler(AsyncioTestCase):
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
self.stream_manager = StreamManager(self.loop, self.blob_manager, None, self.storage, None, 3.0, 3.0)
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))