Compare commits

...
19 Commits
Author SHA1 Message Date
Lex Berezhny e01fbc6f39 initial import of android lbrysdk build 2019-01-27 22:45:32 -05:00
Lex Berezhny 60ed4e503f refactored settings_set to support all settings 2019-01-26 00:04:36 -05:00
Lex Berezhny aa1405887e use float instead of decimal for now 2019-01-25 23:20:43 -05:00
Lex Berezhny cced217e43 update doc format for max_key_fee 2019-01-25 23:20:43 -05:00
Lex Berezhny 8a3b8d2df1 max_key_fee in yaml as null 2019-01-25 23:20:43 -05:00
Lex Berezhny be0bd3bdea refactored --max-key-fee to be more ergonomic 2019-01-25 23:20:43 -05:00
Lex Berezhny e01c73a7f8 pylint 2019-01-25 21:00:20 -05:00
Lex Berezhny 134338db4b dropped help, commands and resolve_name commands 2019-01-25 21:00:20 -05:00
Lex Berezhny f47a685734 help message for start command 2019-01-25 20:28:47 -05:00
Lex Berezhny 6e29f7e99b user is shown help when typing lbrynet help 2019-01-25 20:28:47 -05:00
Lex Berezhny 026d93e92a Update .travis.yml
use `stage: build` for the docker builds to get some parallelism.
2019-01-25 17:28:06 -05:00
Lex Berezhny a55802c1de v0.31.0rc3 2019-01-25 17:05:38 -05:00
Jack Robison 894397a327 Merge pull request #1810 from lbryio/async-reflector
async reflector
2019-01-25 16:41:37 -05:00
Jack Robison 3e3f6c0e22 fix test 2019-01-25 16:20:19 -05:00
Jack Robison a7610e3d34 async reflector 2019-01-25 16:10:17 -05:00
Jack Robison d0e6e74e1b Merge pull request #1809 from lbryio/fix-blob-server
fix blob server protocol factory
2019-01-25 16:07:07 -05:00
Jack Robison 126d71978b disable slow test 2019-01-25 15:38:38 -05:00
Jack Robison 69bcb052de fix blob server protocol factory 2019-01-25 13:10:40 -05:00
Victor Shyba 071af12d1a release wallet server master tag every master merge 2019-01-25 12:20:42 -05:00
47 changed files with 3135 additions and 225 deletions

No files matched your search

+10 -2
View File
@@ -117,13 +117,21 @@ jobs:
- pip3 install -e .
- if: tag IS present
stage: docker
name: "Wallet Server Docker Image"
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:master
cache:
directories:
+5
View File
@@ -0,0 +1,5 @@
build:
python setup.py apk
adb install -r lbry-0.1-service-debug.apk
.PHONY: build
+49
View File
@@ -0,0 +1,49 @@
import inspect
from os.path import dirname, join
import sh
from pythonforandroid.util import ensure_dir
from pythonforandroid.toolchain import (
Bootstrap, current_directory, info, info_main, shprint
)
class LBRYServiceBootstrap(Bootstrap):
name = 'lbry-service'
recipe_depends = ['genericndkbuild', 'python3']
bootstrap_dir = dirname(__file__)
def get_common_dir(self):
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
def run_distribute(self):
info_main('# Creating Android project from build and {} bootstrap'.format(
self.name))
info('This currently just copies the build stuff straight from the build dir.')
shprint(sh.rm, '-rf', self.dist_dir)
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Copying python distribution')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
python_bundle_dir = join('_python_bundle', '_python_bundle')
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super().run_distribute()
+89
View File
@@ -0,0 +1,89 @@
# prevent user to include invalid extensions
*.apk
*.pxd
# eggs
*.egg-info
# unit test
unittest/*
# python config
config/makesetup
# unused kivy files (platform specific)
kivy/input/providers/wm_*
kivy/input/providers/mactouch*
kivy/input/providers/probesysfs*
kivy/input/providers/mtdev*
kivy/input/providers/hidinput*
kivy/core/camera/camera_videocapture*
kivy/core/spelling/*osx*
kivy/core/video/video_pyglet*
kivy/tools
kivy/tests/*
kivy/*/*.h
kivy/*/*.pxi
# unused encodings
lib-dynload/*codec*
encodings/cp*.pyo
encodings/tis*
encodings/shift*
encodings/bz2*
encodings/iso*
encodings/undefined*
encodings/johab*
encodings/p*
encodings/m*
encodings/euc*
encodings/k*
encodings/unicode_internal*
encodings/quo*
encodings/gb*
encodings/big5*
encodings/hp*
encodings/hz*
# unused python modules
bsddb/*
wsgiref/*
hotshot/*
pydoc_data/*
tty.pyo
anydbm.pyo
nturl2path.pyo
LICENCE.txt
macurl2path.pyo
dummy_threading.pyo
audiodev.pyo
antigravity.pyo
dumbdbm.pyo
sndhdr.pyo
__phello__.foo.pyo
sunaudio.pyo
os2emxpath.pyo
multiprocessing/dummy*
# unused binaries python modules
lib-dynload/termios.so
lib-dynload/_lsprof.so
lib-dynload/*audioop.so
lib-dynload/_hotshot.so
lib-dynload/_heapq.so
lib-dynload/_json.so
lib-dynload/grp.so
lib-dynload/resource.so
lib-dynload/pyexpat.so
lib-dynload/_ctypes_test.so
lib-dynload/_testcapi.so
# odd files
plat-linux3/regen
#>sqlite3
# conditionnal include depending if some recipes are included or not.
sqlite3/*
lib-dynload/_sqlite3.so
#<sqlite3
+1
View File
@@ -0,0 +1 @@
include $(call all-subdir-makefiles)
@@ -0,0 +1,7 @@
# Uncomment this if you're using STL in your project
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
# APP_STL := stlport_static
# APP_ABI := armeabi armeabi-v7a x86
APP_ABI := $(ARCH)
@@ -0,0 +1,22 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
# Add your application source files here...
LOCAL_SRC_FILES := start.c pyjniusjni.c
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
LOCAL_SHARED_LIBRARIES := python_shared
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
include $(BUILD_SHARED_LIBRARY)
ifdef CRYSTAX_PYTHON_VERSION
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
endif
@@ -0,0 +1,10 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
LOCAL_SRC_FILES := YourSourceHere.c
include $(BUILD_SHARED_LIBRARY)
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
@@ -0,0 +1,6 @@
#define BOOTSTRAP_NAME_SERVICEONLY
#define BOOTSTRAP_USES_NO_SDL_HEADERS
const char bootstrap_name[] = "service_only";
@@ -0,0 +1,103 @@
#include <pthread.h>
#include <jni.h>
#define LOGI(...) do {} while (0)
#define LOGE(...) do {} while (0)
#include "android/log.h"
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
/* #define LOGP(x) LOG("python", (x)) */
#define LOG_TAG "Python_android"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
/* Function headers */
JNIEnv* Android_JNI_GetEnv(void);
static void Android_JNI_ThreadDestroyed(void*);
static pthread_key_t mThreadKey;
static JavaVM* mJavaVM;
int Android_JNI_SetupThread(void)
{
Android_JNI_GetEnv();
return 1;
}
/* Library init */
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
{
JNIEnv *env;
mJavaVM = vm;
LOGI("JNI_OnLoad called");
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
LOGE("Failed to get the environment using GetEnv()");
return -1;
}
/*
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
*/
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
}
Android_JNI_SetupThread();
return JNI_VERSION_1_4;
}
JNIEnv* Android_JNI_GetEnv(void)
{
/* From http://developer.android.com/guide/practices/jni.html
* All threads are Linux threads, scheduled by the kernel.
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
* and cannot make JNI calls.
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
* is a no-op.
* Note: You can call this function any number of times for the same thread, there's no harm in it
*/
JNIEnv *env;
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
if(status < 0) {
LOGE("failed to attach current thread");
return 0;
}
/* From http://developer.android.com/guide/practices/jni.html
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
* Note: The destructor is not called unless the stored value is != NULL
* Note: You can call this function any number of times for the same thread, there's no harm in it
* (except for some lost CPU cycles)
*/
pthread_setspecific(mThreadKey, (void*) env);
return env;
}
static void Android_JNI_ThreadDestroyed(void* value)
{
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
JNIEnv *env = (JNIEnv*) value;
if (env != NULL) {
(*mJavaVM)->DetachCurrentThread(mJavaVM);
pthread_setspecific(mThreadKey, NULL);
}
}
void *WebView_AndroidGetJNIEnv()
{
return Android_JNI_GetEnv();
}
Whitespace-only changes.
@@ -0,0 +1,141 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class Octal {
/**
* Parse an octal string from a header buffer. This is used for the file
* permission mode value.
*
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal string.
*/
public static long parseOctal(byte[] header, int offset, int length) {
long result = 0;
boolean stillPadding = true;
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
if (header[i] == (byte) ' ' || header[i] == '0') {
if (stillPadding)
continue;
if (header[i] == (byte) ' ')
break;
}
stillPadding = false;
result = ( result << 3 ) + ( header[i] - '0' );
}
return result;
}
/**
* Parse an octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The integer value of the octal bytes.
*/
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
int idx = length - 1;
buf[offset + idx] = 0;
--idx;
buf[offset + idx] = (byte) ' ';
--idx;
if (value == 0) {
buf[offset + idx] = (byte) '0';
--idx;
} else {
for (long val = value; idx >= 0 && val > 0; --idx) {
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
val = val >> 3;
}
}
for (; idx >= 0; --idx) {
buf[offset + idx] = (byte) ' ';
}
return offset + length;
}
/**
* Parse the checksum octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The integer value of the entry's checksum.
*/
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
getOctalBytes( value, buf, offset, length );
buf[offset + length - 1] = (byte) ' ';
buf[offset + length - 2] = 0;
return offset + length;
}
/**
* Parse an octal long integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal bytes.
*/
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
byte[] temp = new byte[length + 1];
getOctalBytes( value, temp, 0, length + 1 );
System.arraycopy( temp, 0, buf, offset, length );
return offset + length;
}
}
@@ -0,0 +1,28 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class TarConstants {
public static final int EOF_BLOCK = 1024;
public static final int DATA_BLOCK = 512;
public static final int HEADER_BLOCK = 512;
}
@@ -0,0 +1,284 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
import java.util.Date;
/**
* @author Kamran Zafar
*
*/
public class TarEntry {
protected File file;
protected TarHeader header;
private TarEntry() {
this.file = null;
header = new TarHeader();
}
public TarEntry(File file, String entryName) {
this();
this.file = file;
this.extractTarHeader(entryName);
}
public TarEntry(byte[] headerBuf) {
this();
this.parseTarHeader(headerBuf);
}
/**
* Constructor to create an entry from an existing TarHeader object.
*
* This method is useful to add new entries programmatically (e.g. for
* adding files or directories that do not exist in the file system).
*
* @param header
*
*/
public TarEntry(TarHeader header) {
this.file = null;
this.header = header;
}
public boolean equals(TarEntry it) {
return header.name.toString().equals(it.header.name.toString());
}
public boolean isDescendent(TarEntry desc) {
return desc.header.name.toString().startsWith(header.name.toString());
}
public TarHeader getHeader() {
return header;
}
public String getName() {
String name = header.name.toString();
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
name = header.namePrefix.toString() + "/" + name;
}
return name;
}
public void setName(String name) {
header.name = new StringBuffer(name);
}
public int getUserId() {
return header.userId;
}
public void setUserId(int userId) {
header.userId = userId;
}
public int getGroupId() {
return header.groupId;
}
public void setGroupId(int groupId) {
header.groupId = groupId;
}
public String getUserName() {
return header.userName.toString();
}
public void setUserName(String userName) {
header.userName = new StringBuffer(userName);
}
public String getGroupName() {
return header.groupName.toString();
}
public void setGroupName(String groupName) {
header.groupName = new StringBuffer(groupName);
}
public void setIds(int userId, int groupId) {
this.setUserId(userId);
this.setGroupId(groupId);
}
public void setModTime(long time) {
header.modTime = time / 1000;
}
public void setModTime(Date time) {
header.modTime = time.getTime() / 1000;
}
public Date getModTime() {
return new Date(header.modTime * 1000);
}
public File getFile() {
return this.file;
}
public long getSize() {
return header.size;
}
public void setSize(long size) {
header.size = size;
}
/**
* Checks if the org.kamrazafar.jtar entry is a directory
*
* @return
*/
public boolean isDirectory() {
if (this.file != null)
return this.file.isDirectory();
if (header != null) {
if (header.linkFlag == TarHeader.LF_DIR)
return true;
if (header.name.toString().endsWith("/"))
return true;
}
return false;
}
/**
* Extract header from File
*
* @param entryName
*/
public void extractTarHeader(String entryName) {
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
}
/**
* Calculate checksum
*
* @param buf
* @return
*/
public long computeCheckSum(byte[] buf) {
long sum = 0;
for (int i = 0; i < buf.length; ++i) {
sum += 255 & buf[i];
}
return sum;
}
/**
* Writes the header to the byte buffer
*
* @param outbuf
*/
public void writeEntryHeader(byte[] outbuf) {
int offset = 0;
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
long size = header.size;
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
int csOffset = offset;
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
outbuf[offset++] = (byte) ' ';
outbuf[offset++] = header.linkFlag;
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
for (; offset < outbuf.length;)
outbuf[offset++] = 0;
long checkSum = this.computeCheckSum(outbuf);
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
}
/**
* Parses the tar header to the byte buffer
*
* @param header
* @param bh
*/
public void parseTarHeader(byte[] bh) {
int offset = 0;
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
offset += TarHeader.MODELEN;
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
offset += TarHeader.UIDLEN;
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
offset += TarHeader.GIDLEN;
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
offset += TarHeader.SIZELEN;
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
offset += TarHeader.MODTIMELEN;
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
offset += TarHeader.CHKSUMLEN;
header.linkFlag = bh[offset++];
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
offset += TarHeader.USTAR_MAGICLEN;
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
offset += TarHeader.USTAR_USER_NAMELEN;
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset += TarHeader.USTAR_GROUP_NAMELEN;
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
}
}
@@ -0,0 +1,243 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* Header
*
* <pre>
* Offset Size Field
* 0 100 File name
* 100 8 File mode
* 108 8 Owner's numeric user ID
* 116 8 Group's numeric user ID
* 124 12 File size in bytes
* 136 12 Last modification time in numeric Unix time format
* 148 8 Checksum for header block
* 156 1 Link indicator (file type)
* 157 100 Name of linked file
* </pre>
*
*
* File Types
*
* <pre>
* Value Meaning
* '0' Normal file
* (ASCII NUL) Normal file (now obsolete)
* '1' Hard link
* '2' Symbolic link
* '3' Character special
* '4' Block special
* '5' Directory
* '6' FIFO
* '7' Contigous
* </pre>
*
*
*
* Ustar header
*
* <pre>
* Offset Size Field
* 257 6 UStar indicator "ustar"
* 263 2 UStar version "00"
* 265 32 Owner user name
* 297 32 Owner group name
* 329 8 Device major number
* 337 8 Device minor number
* 345 155 Filename prefix
* </pre>
*/
public class TarHeader {
/*
* Header
*/
public static final int NAMELEN = 100;
public static final int MODELEN = 8;
public static final int UIDLEN = 8;
public static final int GIDLEN = 8;
public static final int SIZELEN = 12;
public static final int MODTIMELEN = 12;
public static final int CHKSUMLEN = 8;
public static final byte LF_OLDNORM = 0;
/*
* File Types
*/
public static final byte LF_NORMAL = (byte) '0';
public static final byte LF_LINK = (byte) '1';
public static final byte LF_SYMLINK = (byte) '2';
public static final byte LF_CHR = (byte) '3';
public static final byte LF_BLK = (byte) '4';
public static final byte LF_DIR = (byte) '5';
public static final byte LF_FIFO = (byte) '6';
public static final byte LF_CONTIG = (byte) '7';
/*
* Ustar header
*/
public static final String USTAR_MAGIC = "ustar"; // POSIX
public static final int USTAR_MAGICLEN = 8;
public static final int USTAR_USER_NAMELEN = 32;
public static final int USTAR_GROUP_NAMELEN = 32;
public static final int USTAR_DEVLEN = 8;
public static final int USTAR_FILENAME_PREFIX = 155;
// Header values
public StringBuffer name;
public int mode;
public int userId;
public int groupId;
public long size;
public long modTime;
public int checkSum;
public byte linkFlag;
public StringBuffer linkName;
public StringBuffer magic; // ustar indicator and version
public StringBuffer userName;
public StringBuffer groupName;
public int devMajor;
public int devMinor;
public StringBuffer namePrefix;
public TarHeader() {
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
this.name = new StringBuffer();
this.linkName = new StringBuffer();
String user = System.getProperty("user.name", "");
if (user.length() > 31)
user = user.substring(0, 31);
this.userId = 0;
this.groupId = 0;
this.userName = new StringBuffer(user);
this.groupName = new StringBuffer("");
this.namePrefix = new StringBuffer();
}
/**
* Parse an entry name from a header buffer.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The header's entry name.
*/
public static StringBuffer parseName(byte[] header, int offset, int length) {
StringBuffer result = new StringBuffer(length);
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
result.append((char) header[i]);
}
return result;
}
/**
* Determine the number of bytes in an entry name.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The number of bytes in a header's entry name.
*/
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
int i;
for (i = 0; i < length && i < name.length(); ++i) {
buf[offset + i] = (byte) name.charAt(i);
}
for (; i < length; ++i) {
buf[offset + i] = 0;
}
return offset + length;
}
/**
* Creates a new header for a file/directory entry.
*
*
* @param name
* File name
* @param size
* File size in bytes
* @param modTime
* Last modification time in numeric Unix time format
* @param dir
* Is directory
*
* @return
*/
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
String name = entryName;
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
TarHeader header = new TarHeader();
header.linkName = new StringBuffer("");
if (name.length() > 100) {
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
} else {
header.name = new StringBuffer(name);
}
if (dir) {
header.mode = 040755;
header.linkFlag = TarHeader.LF_DIR;
if (header.name.charAt(header.name.length() - 1) != '/') {
header.name.append("/");
}
header.size = 0;
} else {
header.mode = 0100644;
header.linkFlag = TarHeader.LF_NORMAL;
header.size = size;
}
header.modTime = modTime;
header.checkSum = 0;
header.devMajor = 0;
header.devMinor = 0;
return header;
}
}
@@ -0,0 +1,249 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* @author Kamran Zafar
*
*/
public class TarInputStream extends FilterInputStream {
private static final int SKIP_BUFFER_SIZE = 2048;
private TarEntry currentEntry;
private long currentFileSize;
private long bytesRead;
private boolean defaultSkip = false;
public TarInputStream(InputStream in) {
super(in);
currentFileSize = 0;
bytesRead = 0;
}
@Override
public boolean markSupported() {
return false;
}
/**
* Not supported
*
*/
@Override
public synchronized void mark(int readlimit) {
}
/**
* Not supported
*
*/
@Override
public synchronized void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
/**
* Read a byte
*
* @see java.io.FilterInputStream#read()
*/
@Override
public int read() throws IOException {
byte[] buf = new byte[1];
int res = this.read(buf, 0, 1);
if (res != -1) {
return 0xFF & buf[0];
}
return res;
}
/**
* Checks if the bytes being read exceed the entry size and adjusts the byte
* array length. Updates the byte counters
*
*
* @see java.io.FilterInputStream#read(byte[], int, int)
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (currentEntry != null) {
if (currentFileSize == currentEntry.getSize()) {
return -1;
} else if ((currentEntry.getSize() - currentFileSize) < len) {
len = (int) (currentEntry.getSize() - currentFileSize);
}
}
int br = super.read(b, off, len);
if (br != -1) {
if (currentEntry != null) {
currentFileSize += br;
}
bytesRead += br;
}
return br;
}
/**
* Returns the next entry in the tar file
*
* @return TarEntry
* @throws IOException
*/
public TarEntry getNextEntry() throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
int tr = 0;
// Read full header
while (tr < TarConstants.HEADER_BLOCK) {
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
if (res < 0) {
break;
}
System.arraycopy(theader, 0, header, tr, res);
tr += res;
}
// Check if record is null
boolean eof = true;
for (byte b : header) {
if (b != 0) {
eof = false;
break;
}
}
if (!eof) {
currentEntry = new TarEntry(header);
}
return currentEntry;
}
/**
* Returns the current offset (in bytes) from the beginning of the stream.
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
*/
public long getCurrentOffset() {
return bytesRead;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
// Not fully read, skip rest of the bytes
long bs = 0;
while (bs < currentEntry.getSize() - currentFileSize) {
long res = skip(currentEntry.getSize() - currentFileSize - bs);
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
// I suspect file corruption
throw new IOException("Possible tar file corruption");
}
bs += res;
}
}
currentEntry = null;
currentFileSize = 0L;
skipPad();
}
}
/**
* Skips the pad at the end of each tar entry file content
*
* @throws IOException
*/
protected void skipPad() throws IOException {
if (bytesRead > 0) {
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
if (extra > 0) {
long bs = 0;
while (bs < TarConstants.DATA_BLOCK - extra) {
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
bs += res;
}
}
}
}
/**
* Skips 'n' bytes on the InputStream<br>
* Overrides default implementation of skip
*
*/
@Override
public long skip(long n) throws IOException {
if (defaultSkip) {
// use skip method of parent stream
// may not work if skip not implemented by parent
long bs = super.skip(n);
bytesRead += bs;
return bs;
}
if (n <= 0) {
return 0;
}
long left = n;
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
while (left > 0) {
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
if (res < 0) {
break;
}
left -= res;
}
return n - left;
}
public boolean isDefaultSkip() {
return defaultSkip;
}
public void setDefaultSkip(boolean defaultSkip) {
this.defaultSkip = defaultSkip;
}
}
@@ -0,0 +1,163 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
/**
* @author Kamran Zafar
*
*/
public class TarOutputStream extends OutputStream {
private final OutputStream out;
private long bytesWritten;
private long currentFileSize;
private TarEntry currentEntry;
public TarOutputStream(OutputStream out) {
this.out = out;
bytesWritten = 0;
currentFileSize = 0;
}
public TarOutputStream(final File fout) throws FileNotFoundException {
this.out = new BufferedOutputStream(new FileOutputStream(fout));
bytesWritten = 0;
currentFileSize = 0;
}
/**
* Opens a file for writing.
*/
public TarOutputStream(final File fout, final boolean append) throws IOException {
@SuppressWarnings("resource")
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
final long fileSize = fout.length();
if (append && fileSize > TarConstants.EOF_BLOCK) {
raf.seek(fileSize - TarConstants.EOF_BLOCK);
}
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
}
/**
* Appends the EOF record and closes the stream
*
* @see java.io.FilterOutputStream#close()
*/
@Override
public void close() throws IOException {
closeCurrentEntry();
write( new byte[TarConstants.EOF_BLOCK] );
out.close();
}
/**
* Writes a byte to the stream and updates byte counters
*
* @see java.io.FilterOutputStream#write(int)
*/
@Override
public void write(int b) throws IOException {
out.write( b );
bytesWritten += 1;
if (currentEntry != null) {
currentFileSize += 1;
}
}
/**
* Checks if the bytes being written exceed the current entry size.
*
* @see java.io.FilterOutputStream#write(byte[], int, int)
*/
@Override
public void write(byte[] b, int off, int len) throws IOException {
if (currentEntry != null && !currentEntry.isDirectory()) {
if (currentEntry.getSize() < currentFileSize + len) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
+ "] being written." );
}
}
out.write( b, off, len );
bytesWritten += len;
if (currentEntry != null) {
currentFileSize += len;
}
}
/**
* Writes the next tar entry header on the stream
*
* @param entry
* @throws IOException
*/
public void putNextEntry(TarEntry entry) throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
entry.writeEntryHeader( header );
write( header );
currentEntry = entry;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
+ currentEntry.getSize() + "] has not been fully written." );
}
currentEntry = null;
currentFileSize = 0;
pad();
}
}
/**
* Pads the last content block
*
* @throws IOException
*/
protected void pad() throws IOException {
if (bytesWritten > 0) {
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
if (extra > 0) {
write( new byte[TarConstants.DATA_BLOCK - extra] );
}
}
}
}
@@ -0,0 +1,96 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* @author Kamran
*
*/
public class TarUtils {
/**
* Determines the tar file size of the given folder/file path
*
* @param path
* @return
*/
public static long calculateTarSize(File path) {
return tarSize(path) + TarConstants.EOF_BLOCK;
}
private static long tarSize(File dir) {
long size = 0;
if (dir.isFile()) {
return entrySize(dir.length());
} else {
File[] subFiles = dir.listFiles();
if (subFiles != null && subFiles.length > 0) {
for (File file : subFiles) {
if (file.isFile()) {
size += entrySize(file.length());
} else {
size += tarSize(file);
}
}
} else {
// Empty folder header
return TarConstants.HEADER_BLOCK;
}
}
return size;
}
private static long entrySize(long fileSize) {
long size = 0;
size += TarConstants.HEADER_BLOCK; // Header
size += fileSize; // File size
long extra = size % TarConstants.DATA_BLOCK;
if (extra > 0) {
size += (TarConstants.DATA_BLOCK - extra); // pad
}
return size;
}
public static String trim(String s, char c) {
StringBuffer tmp = new StringBuffer(s);
for (int i = 0; i < tmp.length(); i++) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
for (int i = tmp.length() - 1; i >= 0; i--) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
return tmp.toString();
}
}
@@ -0,0 +1,400 @@
package org.kivy.android;
import java.net.Socket;
import java.net.InetSocketAddress;
import android.os.SystemClock;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
import android.app.*;
import android.content.*;
import android.view.*;
import android.view.SurfaceView;
import android.app.Activity;
import android.content.Intent;
import android.util.Log;
import android.widget.Toast;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.PowerManager;
import android.graphics.PixelFormat;
import android.view.SurfaceHolder;
import android.content.Context;
import android.content.pm.ActivityInfo;
import android.content.pm.PackageManager;
import android.content.pm.ApplicationInfo;
import android.content.Intent;
import android.widget.ImageView;
import java.io.InputStream;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Color;
import android.widget.AbsoluteLayout;
import android.webkit.WebViewClient;
import android.webkit.WebView;
import org.kivy.android.PythonUtil;
import org.renpy.android.ResourceManager;
import org.renpy.android.AssetExtract;
public class PythonActivity extends Activity {
// This activity is modified from a mixture of the SDLActivity and
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
// specifics.
private static final String TAG = "PythonActivity";
public static PythonActivity mActivity = null;
/** If shared libraries (e.g. the native application) could not be loaded. */
public static boolean mBrokenLibraries;
protected static Thread mPythonThread;
private ResourceManager resourceManager = null;
private Bundle mMetaData = null;
private PowerManager.WakeLock mWakeLock = null;
public String getAppRoot() {
String app_root = getFilesDir().getAbsolutePath() + "/app";
return app_root;
}
public static void initialize() {
// The static nature of the singleton and Android quirkiness force us to initialize everything here
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
mBrokenLibraries = false;
}
@Override
protected void onCreate(Bundle savedInstanceState) {
Log.v(TAG, "My oncreate running");
resourceManager = new ResourceManager(this);
Log.v(TAG, "Ready to unpack");
File app_root_file = new File(getAppRoot());
unpackData("private", app_root_file);
Log.v(TAG, "About to do super onCreate");
super.onCreate(savedInstanceState);
Log.v(TAG, "Did super onCreate");
this.mActivity = this;
//this.showLoadingScreen();
Log.v("Python", "Device: " + android.os.Build.DEVICE);
Log.v("Python", "Model: " + android.os.Build.MODEL);
//Log.v(TAG, "Ready to unpack");
//new UnpackFilesTask().execute(getAppRoot());
PythonActivity.initialize();
// Load shared libraries
String errorMsgBrokenLib = "";
try {
loadLibraries();
} catch(UnsatisfiedLinkError e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
} catch(Exception e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
}
if (mBrokenLibraries)
{
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
+ System.getProperty("line.separator")
+ System.getProperty("line.separator")
+ "Error: " + errorMsgBrokenLib);
dlgAlert.setTitle("Python Error");
dlgAlert.setPositiveButton("Exit",
new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog,int id) {
// if this button is clicked, close current activity
PythonActivity.mActivity.finish();
}
});
dlgAlert.setCancelable(false);
dlgAlert.create().show();
return;
}
// Set up the Python environment
String app_root_dir = getAppRoot();
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
Log.v(TAG, "Setting env vars for start.c and Python to use");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
try {
Log.v(TAG, "Access to our meta-data...");
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
mActivity.mWakeLock.acquire();
}
} catch (PackageManager.NameNotFoundException e) {
}
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
PythonActivity.mPythonThread = pythonThread;
pythonThread.start();
}
@Override
public void onDestroy() {
Log.i("Destroy", "end of app");
super.onDestroy();
// make sure all child threads (python_thread) are stopped
android.os.Process.killProcess(android.os.Process.myPid());
}
public void loadLibraries() {
String app_root = new String(getAppRoot());
File app_root_file = new File(app_root);
PythonUtil.loadLibraries(app_root_file);
}
public void recursiveDelete(File f) {
if (f.isDirectory()) {
for (File r : f.listFiles()) {
recursiveDelete(r);
}
}
f.delete();
}
/**
* Show an error using a toast. (Only makes sense from non-UI
* threads.)
*/
public void toastError(final String msg) {
final Activity thisActivity = this;
runOnUiThread(new Runnable () {
public void run() {
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
}
});
// Wait to show the error.
synchronized (this) {
try {
this.wait(1000);
} catch (InterruptedException e) {
}
}
}
public void unpackData(final String resource, File target) {
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
// The version of data in memory and on disk.
String data_version = resourceManager.getString(resource + "_version");
String disk_version = null;
Log.v(TAG, "Data version is " + data_version);
// If no version, no unpacking is necessary.
if (data_version == null) {
return;
}
// Check the current disk version, if any.
String filesDir = target.getAbsolutePath();
String disk_version_fn = filesDir + "/" + resource + ".version";
try {
byte buf[] = new byte[64];
InputStream is = new FileInputStream(disk_version_fn);
int len = is.read(buf);
disk_version = new String(buf, 0, len);
is.close();
} catch (Exception e) {
disk_version = "";
}
// If the disk data is out of date, extract it and write the
// version file.
// if (! data_version.equals(disk_version)) {
if (! data_version.equals(disk_version)) {
Log.v(TAG, "Extracting " + resource + " assets.");
recursiveDelete(target);
target.mkdirs();
AssetExtract ae = new AssetExtract(this);
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
toastError("Could not extract " + resource + " data.");
}
try {
// Write .nomedia.
new File(target, ".nomedia").createNewFile();
// Write version file.
FileOutputStream os = new FileOutputStream(disk_version_fn);
os.write(data_version.getBytes());
os.close();
} catch (Exception e) {
Log.w("python", e);
}
}
}
long lastBackClick = SystemClock.elapsedRealtime();
@Override
public boolean onKeyDown(int keyCode, KeyEvent event) {
// If it wasn't the Back key or there's no web page history, bubble up to the default
// system behavior (probably exit the activity)
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
lastBackClick = SystemClock.elapsedRealtime();
Toast.makeText(this, "Click again to close the app",
Toast.LENGTH_LONG).show();
return true;
}
lastBackClick = SystemClock.elapsedRealtime();
return super.onKeyDown(keyCode, event);
}
//----------------------------------------------------------------------------
// Listener interface for onNewIntent
//
public interface NewIntentListener {
void onNewIntent(Intent intent);
}
private List<NewIntentListener> newIntentListeners = null;
public void registerNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
this.newIntentListeners.add(listener);
}
public void unregisterNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
return;
this.newIntentListeners.remove(listener);
}
@Override
protected void onNewIntent(Intent intent) {
if ( this.newIntentListeners == null )
return;
this.onResume();
synchronized ( this.newIntentListeners ) {
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
while ( iterator.hasNext() ) {
(iterator.next()).onNewIntent(intent);
}
}
}
//----------------------------------------------------------------------------
// Listener interface for onActivityResult
//
public interface ActivityResultListener {
void onActivityResult(int requestCode, int resultCode, Intent data);
}
private List<ActivityResultListener> activityResultListeners = null;
public void registerActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
this.activityResultListeners.add(listener);
}
public void unregisterActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
return;
this.activityResultListeners.remove(listener);
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
if ( this.activityResultListeners == null )
return;
this.onResume();
synchronized ( this.activityResultListeners ) {
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
while ( iterator.hasNext() )
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
}
}
public static void start_service(String serviceTitle, String serviceDescription,
String pythonServiceArgument) {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
serviceIntent.putExtra("serviceTitle", serviceTitle);
serviceIntent.putExtra("serviceDescription", serviceDescription);
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
PythonActivity.mActivity.startService(serviceIntent);
}
public static void stop_service() {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
PythonActivity.mActivity.stopService(serviceIntent);
}
public static native void nativeSetEnv(String j_name, String j_value);
public static native int nativeInit(Object arguments);
}
class PythonMain implements Runnable {
@Override
public void run() {
PythonActivity.nativeInit(new String[0]);
}
}
@@ -0,0 +1,45 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonEvent {
private final Lock lock = new ReentrantLock();
private final Condition cond = lock.newCondition();
private boolean flag = false;
public void set() {
lock.lock();
try {
flag = true;
cond.signalAll();
} finally {
lock.unlock();
}
}
public void wait_() throws InterruptedException {
lock.lock();
try {
while (!flag) {
cond.await();
}
} finally {
lock.unlock();
}
}
public void clear() {
lock.lock();
try {
flag = false;
cond.signalAll();
} finally {
lock.unlock();
}
}
}
@@ -0,0 +1,19 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonLock {
private final Lock lock = new ReentrantLock();
public void acquire() {
lock.lock();
}
public void release() {
lock.unlock();
}
}
@@ -0,0 +1,257 @@
package org.renpy.android;
import java.util.List;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
import android.hardware.Sensor;
import android.hardware.SensorEvent;
import android.hardware.SensorEventListener;
import android.hardware.SensorManager;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import android.net.wifi.ScanResult;
import android.net.wifi.WifiManager;
import android.os.Vibrator;
import android.view.View;
/**
* Methods that are expected to be called via JNI, to access the device's
* non-screen hardware. (For example, the vibration and accelerometer.)
*/
public class Hardware {
// The context.
static Context context;
static View view;
/**
* Vibrate for s seconds.
*/
public static void vibrate(double s) {
Vibrator v = (Vibrator) context
.getSystemService(Context.VIBRATOR_SERVICE);
if (v != null) {
v.vibrate((int) (1000 * s));
}
}
/**
* Get an Overview of all Hardware Sensors of an Android Device
*/
public static String getHardwareSensors() {
SensorManager sm = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
if (allSensors != null) {
String resultString = "";
for (Sensor s : allSensors) {
resultString += String.format("Name=" + s.getName());
resultString += String.format(",Vendor=" + s.getVendor());
resultString += String.format(",Version=" + s.getVersion());
resultString += String.format(",MaximumRange="
+ s.getMaximumRange());
// XXX MinDelay is not in the 2.2
// resultString += String.format(",MinDelay=" +
// s.getMinDelay());
resultString += String.format(",Power=" + s.getPower());
resultString += String.format(",Type=" + s.getType() + "\n");
}
return resultString;
}
return "";
}
/**
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
* Magnetic Field Sensors
*/
public static class generic3AxisSensor implements SensorEventListener {
private final SensorManager sSensorManager;
private final Sensor sSensor;
private final int sSensorType;
SensorEvent sSensorEvent;
public generic3AxisSensor(int sensorType) {
sSensorType = sensorType;
sSensorManager = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
sSensor = sSensorManager.getDefaultSensor(sSensorType);
}
public void onAccuracyChanged(Sensor sensor, int accuracy) {
}
public void onSensorChanged(SensorEvent event) {
sSensorEvent = event;
}
/**
* Enable or disable the Sensor by registering/unregistering
*/
public void changeStatus(boolean enable) {
if (enable) {
sSensorManager.registerListener(this, sSensor,
SensorManager.SENSOR_DELAY_NORMAL);
} else {
sSensorManager.unregisterListener(this, sSensor);
}
}
/**
* Read the Sensor
*/
public float[] readSensor() {
if (sSensorEvent != null) {
return sSensorEvent.values;
} else {
float rv[] = { 0f, 0f, 0f };
return rv;
}
}
}
public static generic3AxisSensor accelerometerSensor = null;
public static generic3AxisSensor orientationSensor = null;
public static generic3AxisSensor magneticFieldSensor = null;
/**
* functions for backward compatibility reasons
*/
public static void accelerometerEnable(boolean enable) {
if (accelerometerSensor == null)
accelerometerSensor = new generic3AxisSensor(
Sensor.TYPE_ACCELEROMETER);
accelerometerSensor.changeStatus(enable);
}
public static float[] accelerometerReading() {
float rv[] = { 0f, 0f, 0f };
if (accelerometerSensor == null)
return rv;
return (float[]) accelerometerSensor.readSensor();
}
public static void orientationSensorEnable(boolean enable) {
if (orientationSensor == null)
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
orientationSensor.changeStatus(enable);
}
public static float[] orientationSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (orientationSensor == null)
return rv;
return (float[]) orientationSensor.readSensor();
}
public static void magneticFieldSensorEnable(boolean enable) {
if (magneticFieldSensor == null)
magneticFieldSensor = new generic3AxisSensor(
Sensor.TYPE_MAGNETIC_FIELD);
magneticFieldSensor.changeStatus(enable);
}
public static float[] magneticFieldSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (magneticFieldSensor == null)
return rv;
return (float[]) magneticFieldSensor.readSensor();
}
/**
* Scan WiFi networks
*/
static List<ScanResult> latestResult;
public static void enableWifiScanner() {
IntentFilter i = new IntentFilter();
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
// occurs
WifiManager w = (WifiManager) c
.getSystemService(Context.WIFI_SERVICE);
latestResult = w.getScanResults(); // Returns a <list> of
// scanResults
}
}, i);
}
public static String scanWifi() {
// Now you can call this and it should execute the broadcastReceiver's
// onReceive()
WifiManager wm = (WifiManager) context
.getSystemService(Context.WIFI_SERVICE);
boolean a = wm.startScan();
if (latestResult != null) {
String latestResultString = "";
for (ScanResult result : latestResult) {
latestResultString += String.format("%s\t%s\t%d\n",
result.SSID, result.BSSID, result.level);
}
return latestResultString;
}
return "";
}
/**
* network state
*/
public static boolean network_state = false;
/**
* Check network state directly
*
* (only one connection can be active at a given moment, detects all network
* type)
*
*/
public static boolean checkNetwork() {
boolean state = false;
final ConnectivityManager conMgr = (ConnectivityManager) context
.getSystemService(Context.CONNECTIVITY_SERVICE);
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
if (activeNetwork != null && activeNetwork.isConnected()) {
state = true;
} else {
state = false;
}
return state;
}
/**
* To recieve network state changes
*/
public static void registerNetworkCheck() {
IntentFilter i = new IntentFilter();
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
network_state = checkNetwork();
}
}, i);
}
}
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,93 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="{{ args.package }}"
android:versionCode="{{ args.numeric_version }}"
android:versionName="{{ args.version }}"
android:installLocation="auto">
<supports-screens
android:smallScreens="true"
android:normalScreens="true"
android:largeScreens="true"
android:anyDensity="true"
{% if args.min_sdk_version >= 9 %}
android:xlargeScreens="true"
{% endif %}
/>
<!-- Android 2.3.3 -->
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
<!-- Set permissions -->
{% for perm in args.permissions %}
{% if '.' in perm %}
<uses-permission android:name="{{ perm }}" />
{% else %}
<uses-permission android:name="android.permission.{{ perm }}" />
{% endif %}
{% endfor %}
{% if args.wakelock %}
<uses-permission android:name="android.permission.WAKE_LOCK" />
{% endif %}
{% if args.billing_pubkey %}
<uses-permission android:name="com.android.vending.BILLING" />
{% endif %}
<!-- Create a Java class extending SDLActivity and place it in a
directory under src matching the package, e.g.
src/com/gamemaker/game/MyGame.java
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
in the XML below.
An example Java class can be found in README-android.txt
-->
<application android:label="@string/app_name"
android:icon="@drawable/icon"
android:allowBackup="true"
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
android:hardwareAccelerated="true" >
{% for m in args.meta_data %}
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
<activity android:name="org.kivy.android.PythonActivity"
android:label="@string/app_name"
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
>
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
{%- if args.intent_filters -%}
{{- args.intent_filters -}}
{%- endif -%}
</activity>
{% if service %}
<service android:name="org.kivy.android.PythonService"
android:process=":pythonservice" />
{% endif %}
{% for name in service_names %}
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
android:process=":service_{{ name }}" />
{% endfor %}
{% if args.billing_pubkey %}
<service android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbilling" />
<receiver android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbillingreceiver">
<intent-filter>
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
</intent-filter>
</receiver>
{% endif %}
</application>
</manifest>
@@ -0,0 +1,74 @@
package {{ args.package }};
import android.os.Binder;
import android.os.IBinder;
import android.content.Intent;
import android.content.Context;
import org.kivy.android.PythonService;
public class Service{{ name|capitalize }} extends PythonService {
/**
* Binder given to clients
*/
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
{% if sticky %}
/**
* {@inheritDoc}
*/
@Override
public int getStartType() {
return START_STICKY;
}
{% endif %}
{% if foreground %}
/**
* {@inheritDoc}
*/
@Override
public boolean getStartForeground() {
return true;
}
{% endif %}
public static void start(Context ctx, String pythonServiceArgument) {
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
intent.putExtra("androidPrivate", argument);
intent.putExtra("androidArgument", argument);
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
intent.putExtra("serviceDescription", "");
intent.putExtra("pythonName", "{{ name }}");
intent.putExtra("pythonHome", argument);
intent.putExtra("androidUnpack", argument);
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
ctx.startService(intent);
}
public static void stop(Context ctx) {
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
ctx.stopService(intent);
}
/**
* {@inheritDoc}
*/
@Override
public IBinder onBind(Intent intent) {
return mBinder;
}
/**
* Class used for the client Binder. Because we know this service always
* runs in the same process as its clients, we don't need to deal with IPC.
*/
public class Service{{ name|capitalize }}Binder extends Binder {
Service{{ name|capitalize }} getService() {
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
return Service{{ name|capitalize }}.this;
}
}
}
@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string name="app_name">{{ args.name }}</string>
<string name="private_version">{{ private_version }}</string>
</resources>
+68
View File
@@ -0,0 +1,68 @@
from lbrynet.extras.daemon.Daemon import Daemon
from jnius import PythonJavaClass, autoclass, java_method
JAVA_NAMESPACE = 'org.kivy.android'
JNI_NAMESPACE = 'org/kivy/android'
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
_callbacks = {
'on_new_intent': [],
'on_activity_result': [],
}
class NewIntentListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
__javacontext__ = 'app'
def __init__(self, callback, **kwargs):
super(NewIntentListener, self).__init__(**kwargs)
self.callback = callback
@java_method('(Landroid/content/Intent;)V')
def onNewIntent(self, intent):
self.callback(intent)
class ActivityResultListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
__javacontext__ = 'app'
def __init__(self, callback):
super(ActivityResultListener, self).__init__()
self.callback = callback
@java_method('(IILandroid/content/Intent;)V')
def onActivityResult(self, requestCode, resultCode, intent):
self.callback(requestCode, resultCode, intent)
def bind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
elif event == 'on_new_intent':
listener = NewIntentListener(callback)
_activity.registerNewIntentListener(listener)
_callbacks[event].append(listener)
elif event == 'on_activity_result':
listener = ActivityResultListener(callback)
_activity.registerActivityResultListener(listener)
_callbacks[event].append(listener)
def unbind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
else:
for listener in _callbacks[event][:]:
if listener.callback == callback:
_callbacks[event].remove(listener)
if event == 'on_new_intent':
_activity.unregisterNewIntentListener(listener)
elif event == 'on_activity_result':
_activity.unregisterActivityResultListener(listener)
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
+49
View File
@@ -0,0 +1,49 @@
from os.path import join, dirname, abspath
from pythonforandroid.toolchain import Bootstrap
from setuptools import setup, find_packages
from bootstrap import LBRYServiceBootstrap
Bootstrap.bootstraps = {
'lbry-service': LBRYServiceBootstrap()
}
setup(
name='lbryservice',
version='0.1',
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
description="Android Service for LBRY Network.",
license='MIT',
python_requires='>=3.7',
packages=find_packages(),
package_data={'service': ['*.py']},
options={
'apk': {
'dist_name': 'lbry-service',
'bootstrap': 'lbry-service',
'package': 'io.lbry.service',
'permissions': ['INTERNET'],
'requirements': ','.join([
# needed by aiohttp
'multidict', 'yarl', 'async_timeout', 'chardet',
# minimum needed by torba:
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
abspath(join(dirname(__file__), '..', '..', 'torba')),
# minimum needed by lbrynet
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
abspath(join(dirname(__file__), '..')),
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
]),
'android-api': '26',
'ndk-api': '21',
'ndk-version': 'r17c',
'arch': 'armeabi-v7a',
'sdk-dir': '/home/lex/projects/android',
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
}
}
)
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.31.0rc2"
__version__ = "0.31.0rc3"
version = tuple(__version__.split('.'))
+14 -2
View File
@@ -13,7 +13,7 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
class BlobServer(asyncio.Protocol):
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
@@ -87,12 +87,24 @@ class BlobServer(asyncio.Protocol):
return
self.loop.create_task(self.handle_request(request))
class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
raise Exception("already running")
async def _start_server():
server = await self.loop.create_server(lambda: self, interface, port)
server = await self.loop.create_server(
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
interface, port
)
self.started_listening.set()
log.info("Blob server listening on TCP %s:%i", interface, port)
async with server:
+94 -38
View File
@@ -61,6 +61,14 @@ class Setting(typing.Generic[T]):
def __set_name__(self, owner, name):
self.name = name
@property
def cli_name(self):
return f"--{self.name.replace('_', '-')}"
@property
def no_cli_name(self):
return f"--no-{self.name.replace('_', '-')}"
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
if obj is None:
return self
@@ -88,6 +96,14 @@ class Setting(typing.Generic[T]):
def serialize(self, value):
return value
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
metavar=self.metavar,
default=NOT_SET
)
class String(Setting[str]):
def validate(self, val):
@@ -100,18 +116,39 @@ class Integer(Setting[int]):
assert isinstance(val, int), \
f"Setting '{self.name}' must be an integer."
def deserialize(self, value):
return int(value)
class Float(Setting[float]):
def validate(self, val):
assert isinstance(val, float), \
f"Setting '{self.name}' must be a decimal."
def deserialize(self, value):
return float(value)
class Toggle(Setting[bool]):
def validate(self, val):
assert isinstance(val, bool), \
f"Setting '{self.name}' must be a true/false value."
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
action="store_true",
default=NOT_SET
)
parser.add_argument(
self.no_cli_name,
help=f"Opposite of --{self.cli_name}",
dest=self.name,
action="store_false",
default=NOT_SET
)
class Path(String):
def __init__(self, doc: str, default: str = '', *args, **kwargs):
@@ -127,13 +164,54 @@ class Path(String):
class MaxKeyFee(Setting[dict]):
def validate(self, value):
assert isinstance(value, dict), \
f"Setting '{self.name}' must be of the format \"{'currency': 'USD', 'amount': 50.0}\"."
assert set(value) == {'currency', 'amount'}, \
f"Setting '{self.name}' must contain a 'currency' and an 'amount' field."
currency = str(value["currency"]).upper()
if value is not None:
assert isinstance(value, dict) and set(value) == {'currency', 'amount'}, \
f"Setting '{self.name}' must be a dict like \"{{'amount': 50.0, 'currency': 'USD'}}\"."
if value["currency"] not in CURRENCIES:
raise InvalidCurrencyError(value["currency"])
@staticmethod
def _parse_list(l):
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
try:
amount = float(l[0])
except ValueError:
raise AssertionError('First value in max key fee is a decimal: "AMOUNT CURRENCY"')
currency = str(l[1]).upper()
if currency not in CURRENCIES:
raise InvalidCurrencyError(currency)
return {'amount': amount, 'currency': currency}
def deserialize(self, value):
if value is None:
return
if isinstance(value, dict):
return {
'currency': value['currency'],
'amount': float(value['amount']),
}
if isinstance(value, str):
value = value.split()
if isinstance(value, list):
return self._parse_list(value)
raise AssertionError('Invalid max key fee.')
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
nargs=2,
metavar=('AMOUNT', 'CURRENCY'),
default=NOT_SET
)
parser.add_argument(
self.no_cli_name,
help=f"Disable maximum key fee check.",
dest=self.name,
const=None,
action="store_const",
default=NOT_SET
)
class Servers(Setting[list]):
@@ -169,6 +247,14 @@ class Servers(Setting[list]):
return [f"{host}:{port}" for host, port in value]
return value
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
nargs="*",
help=self.doc,
default=NOT_SET
)
class Strings(Setting[list]):
@@ -205,7 +291,7 @@ class ArgumentAccess:
def load(self, args):
for setting in self.configuration.get_settings():
value = getattr(args, setting.name, NOT_SET)
if value not in (None, NOT_SET):
if value != NOT_SET:
self.args[setting.name] = setting.deserialize(value)
def __contains__(self, item: str):
@@ -340,36 +426,9 @@ class BaseConfig:
return conf
@classmethod
def contribute_args(cls, parser: ArgumentParser):
def contribute_to_argparse(cls, parser: ArgumentParser):
for setting in cls.get_settings():
if isinstance(setting, Toggle):
parser.add_argument(
f"--{setting.name.replace('_', '-')}",
help=setting.doc,
action="store_true",
default=NOT_SET
)
parser.add_argument(
f"--no-{setting.name.replace('_', '-')}",
help=f"Opposite of --{setting.name.replace('_', '-')}",
dest=setting.name,
action="store_false",
default=NOT_SET
)
elif isinstance(setting, Servers):
parser.add_argument(
f"--{setting.name.replace('_', '-')}",
nargs="*",
help=setting.doc,
default=None
)
else:
parser.add_argument(
f"--{setting.name.replace('_', '-')}",
help=setting.doc,
metavar=setting.metavar,
default=NOT_SET
)
setting.contribute_to_argparse(parser)
def set_arguments(self, args):
self.arguments = ArgumentAccess(self, args)
@@ -451,10 +510,7 @@ class Config(CLIConfig):
])
max_connections_per_stream = Integer("", 5)
seek_head_blob_first = Toggle("", True)
# TODO: writing json on the cmd line is a pain, come up with a nicer
# parser for this data structure. maybe 'USD:25'
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
disable_max_key_fee = Toggle("", False)
min_info_rate = Float("points/1000 infos", .02)
min_valuable_hash_rate = Float("points/1000 infos", .05)
min_valuable_info_rate = Float("points/1000 infos", .05)
+11 -5
View File
@@ -141,6 +141,10 @@ class ArgumentParser(argparse.ArgumentParser):
self, action, "Commands", lambda parser: 'group' not in parser._defaults
)
def error(self, message):
self.print_help(argparse._sys.stderr)
self.exit(2, '\n'+message+'\n')
class HelpFormatter(argparse.HelpFormatter):
@@ -164,18 +168,20 @@ def add_command_parser(parent, command):
def get_argument_parser():
main = ArgumentParser('lbrynet')
main = ArgumentParser(
'lbrynet', description='An interface to the LBRY Network.'
)
main.add_argument(
'-v', '--version', dest='cli_version', action="store_true",
help='Show lbrynet CLI version and exit.'
)
main.set_defaults(group=None, command=None)
CLIConfig.contribute_args(main)
CLIConfig.contribute_to_argparse(main)
sub = main.add_subparsers(metavar='COMMAND')
start = sub.add_parser(
'start',
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
help='Start lbrynet API server.'
help='Start LBRY Network interface.'
)
start.add_argument(
'--quiet', dest='quiet', action="store_true",
@@ -187,7 +193,7 @@ def get_argument_parser():
'should selectively be applied.')
)
start.set_defaults(command='start', start_parser=start)
Config.contribute_args(start)
Config.contribute_to_argparse(start)
api = Daemon.get_api_definitions()
groups = {}
@@ -195,7 +201,7 @@ def get_argument_parser():
group_parser = sub.add_parser(group_name, group_name=group_name, help=api['groups'][group_name])
groups[group_name] = group_parser.add_subparsers(metavar='COMMAND')
nicer_order = ['stop', 'get', 'publish', 'resolve', 'resolve_name']
nicer_order = ['stop', 'get', 'publish', 'resolve']
for command_name in sorted(api['commands']):
if command_name not in nicer_order:
nicer_order.append(command_name)
+1 -1
View File
@@ -466,7 +466,7 @@ class StreamManagerComponent(Component):
self.conf.peer_connect_timeout, [
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
for url, port in self.conf.reflector_servers
]
], self.conf.reflector_servers
)
await self.stream_manager.start()
log.info('Done setting up file manager')
+8 -126
View File
@@ -3,7 +3,6 @@ import asyncio
import logging
import json
import inspect
import textwrap
import typing
import aiohttp
import base58
@@ -20,7 +19,7 @@ from lbrynet import __version__, utils
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.error import InsufficientFundsError, UnknownNameError, DownloadSDTimeout, ComponentsNotStarted
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
from lbrynet.extras import system_info
from lbrynet.extras.daemon import analytics
@@ -295,8 +294,7 @@ class Daemon(metaclass=JSONRPCServerType):
@classmethod
def get_api_definitions(cls):
prefix = 'jsonrpc_'
skip = ['commands', 'help']
not_grouped = ['block_show', 'report_bug', 'resolve_name', 'routing_table_get']
not_grouped = ['block_show', 'report_bug', 'routing_table_get']
api = {
'groups': {
group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip()
@@ -307,8 +305,6 @@ class Daemon(metaclass=JSONRPCServerType):
for jsonrpc_method in dir(cls):
if jsonrpc_method.startswith(prefix):
full_name = jsonrpc_method[len(prefix):]
if full_name in skip:
continue
method = getattr(cls, jsonrpc_method)
if full_name in not_grouped:
name_parts = [full_name]
@@ -760,109 +756,21 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.conf.settings_dict
def jsonrpc_settings_set(self, **kwargs):
def jsonrpc_settings_set(self, key, value):
"""
Set daemon settings
Usage:
settings_set [--download_directory=<download_directory>]
[--data_rate=<data_rate>]
[--download_timeout=<download_timeout>]
[--peer_port=<peer_port>]
[--max_key_fee=<max_key_fee>]
[--disable_max_key_fee=<disable_max_key_fee>]
[--use_upnp=<use_upnp>]
[--run_reflector_server=<run_reflector_server>]
[--cache_time=<cache_time>]
[--reflect_uploads=<reflect_uploads>]
[--share_usage_data=<share_usage_data>]
[--peer_search_timeout=<peer_search_timeout>]
[--sd_download_timeout=<sd_download_timeout>]
[--auto_renew_claim_height_delta=<auto_renew_claim_height_delta>]
Options:
--download_directory=<download_directory> : (str) path of download directory
--data_rate=<data_rate> : (float) 0.0001
--download_timeout=<download_timeout> : (int) 180
--peer_port=<peer_port> : (int) 3333
--max_key_fee=<max_key_fee> : (dict) maximum key fee for downloads,
in the format:
{
'currency': <currency_symbol>,
'amount': <amount>
}.
In the CLI, it must be an escaped JSON string
Supported currency symbols: LBC, USD, BTC
--disable_max_key_fee=<disable_max_key_fee> : (bool) False
--use_upnp=<use_upnp> : (bool) True
--run_reflector_server=<run_reflector_server> : (bool) False
--cache_time=<cache_time> : (int) 150
--reflect_uploads=<reflect_uploads> : (bool) True
--share_usage_data=<share_usage_data> : (bool) True
--peer_search_timeout=<peer_search_timeout> : (int) 3
--sd_download_timeout=<sd_download_timeout> : (int) 3
--auto_renew_claim_height_delta=<auto_renew_claim_height_delta> : (int) 0
claims set to expire within this many blocks will be
automatically renewed after startup (if set to 0, renews
will not be made automatically)
settings_set <key> <value>
Returns:
(dict) Updated dictionary of daemon settings
"""
with self.conf.update_config() as c:
for key, value in kwargs:
attr: Setting = getattr(type(c), key)
setattr(c, key, attr.deserialize(value))
return self.jsonrpc_settings_get()
def jsonrpc_help(self, command=None):
"""
Return a useful message for an API command
Usage:
help [<command> | --command=<command>]
Options:
--command=<command> : (str) command to retrieve documentation for
Returns:
(str) Help message
"""
if command is None:
return {
'about': 'This is the LBRY JSON-RPC API',
'command_help': 'Pass a `command` parameter to this method to see ' +
'help for that command (e.g. `help command=resolve_name`)',
'command_list': 'Get a full list of commands using the `commands` method',
'more_info': 'Visit https://lbry.io/api for more info',
}
fn = self.callable_methods.get(command)
if fn is None:
raise Exception(
f"No help available for '{command}'. It is not a valid command."
)
return {
'help': textwrap.dedent(fn.__doc__ or '')
}
def jsonrpc_commands(self):
"""
Return a list of available commands
Usage:
commands
Options:
None
Returns:
(list) list of available commands
"""
return sorted([command for command in self.callable_methods.keys()])
attr: Setting = getattr(type(c), key)
cleaned = attr.deserialize(value)
setattr(c, key, cleaned)
return {key: cleaned}
WALLET_DOC = """
Wallet management.
@@ -1460,32 +1368,6 @@ class Daemon(metaclass=JSONRPCServerType):
)
]
@requires(WALLET_COMPONENT)
async def jsonrpc_resolve_name(self, name, force=False):
"""
Resolve stream info from a LBRY name
Usage:
resolve_name (<name> | --name=<name>) [--force]
Options:
--name=<name> : (str) the name to resolve
--force : (bool) force refresh and do not check cache
Returns:
(dict) Metadata dictionary from name claim, None if the name is not
resolvable
"""
try:
name = parse_lbry_uri(name).name
metadata = await self.wallet_manager.resolve(name, check_cache=not force)
if name in metadata:
metadata = metadata[name]
return metadata
except UnknownNameError:
log.info('Name %s is not known', name)
CLAIM_DOC = """
Claim management.
"""
+40 -2
View File
@@ -5,6 +5,7 @@ import logging
from lbrynet.extras.daemon.mime_types import guess_media_type
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.stream.reflector.client import StreamReflectorClient
if typing.TYPE_CHECKING:
from lbrynet.extras.daemon.storage import StoredStreamClaim
from lbrynet.blob.blob_manager import BlobFileManager
@@ -30,6 +31,7 @@ class ManagedStream:
self.stream_claim_info = claim
self._status = status
self._store_after_finished: asyncio.Task = None
self.fully_reflected = asyncio.Event(loop=self.loop)
@property
def status(self) -> str:
@@ -139,9 +141,10 @@ class ManagedStream:
@classmethod
async def create(cls, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager',
file_path: str) -> 'ManagedStream':
file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
descriptor = await StreamDescriptor.create_stream(
loop, blob_manager.blob_dir, file_path
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
)
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
await blob_manager.blob_completed(sd_blob)
@@ -156,3 +159,38 @@ class ManagedStream:
await self.downloader.stop()
if not self.finished:
self.update_status(self.STATUS_STOPPED)
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try:
await self.loop.create_connection(lambda: protocol, host, port)
except ConnectionRefusedError:
return sent
try:
await protocol.send_handshake()
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
try:
sent_sd, needed = await protocol.send_descriptor()
if sent_sd:
sent.append(self.sd_hash)
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
for blob_hash in needed:
try:
await protocol.send_blob(blob_hash)
sent.append(blob_hash)
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
if protocol.transport:
protocol.transport.close()
return sent
if protocol.transport:
protocol.transport.close()
if not self.fully_reflected.is_set():
self.fully_reflected.set()
return sent
Whitespace-only changes.
+103
View File
@@ -0,0 +1,103 @@
import asyncio
import json
import logging
import typing
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.stream.descriptor import StreamDescriptor
REFLECTOR_V1 = 0
REFLECTOR_V2 = 1
log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
self.transport: asyncio.StreamWriter = None
self.blob_manager = blob_manager
self.descriptor = descriptor
self.response_buff = b''
self.reflected_blobs = []
self.connected = asyncio.Event()
self.response_queue = asyncio.Queue(maxsize=1)
self.pending_request: typing.Optional[asyncio.Task] = None
def connection_made(self, transport):
self.transport = transport
log.info("Connected to reflector")
self.connected.set()
def connection_lost(self, exc: typing.Optional[Exception]):
self.transport = None
self.connected.clear()
def data_received(self, data):
try:
response = json.loads(data.decode())
self.response_queue.put_nowait(response)
except ValueError:
self.transport.close()
return
async def send_request(self, request_dict: typing.Dict):
msg = json.dumps(request_dict)
self.transport.write(msg.encode())
try:
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
return await self.pending_request
finally:
self.pending_request = None
async def send_handshake(self) -> None:
response_dict = await self.send_request({'version': REFLECTOR_V2})
if 'version' not in response_dict:
raise ValueError("Need protocol version number!")
server_version = int(response_dict['version'])
if server_version != REFLECTOR_V2:
raise ValueError("I can't handle protocol version {}!".format(server_version))
return
async def send_descriptor(self) -> typing.Tuple[bool, typing.List[str]]: # returns a list of needed blob hashes
sd_blob = self.blob_manager.get_blob(self.descriptor.sd_hash)
assert sd_blob.get_is_verified(), "need to have a sd blob to send at this point"
response = await self.send_request({
'sd_blob_hash': sd_blob.blob_hash,
'sd_blob_size': sd_blob.length
})
if 'send_sd_blob' not in response:
raise ValueError("I don't know whether to send the sd blob or not!")
needed = response.get('needed_blobs', [])
sent_sd = False
if response['send_sd_blob']:
await sd_blob.sendfile(self)
received = await self.response_queue.get()
if received.get('received_sd_blob'):
sent_sd = True
if not needed:
for blob in self.descriptor.blobs[:-1]:
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
needed.append(blob.blob_hash)
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
else:
log.warning("Reflector failed to receive descriptor %s", sd_blob.blob_hash[:8])
return sent_sd, needed
async def send_blob(self, blob_hash: str):
blob = self.blob_manager.get_blob(blob_hash)
assert blob.get_is_verified(), "need to have a blob to send at this point"
response = await self.send_request({
'blob_hash': blob.blob_hash,
'blob_size': blob.length
})
if 'send_blob' not in response:
raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']:
await blob.sendfile(self)
received = await self.response_queue.get()
if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8])
else:
log.warning("Reflector failed to receive blob %s", blob.blob_hash[:8])
+147
View File
@@ -0,0 +1,147 @@
import asyncio
import logging
import typing
import json
from json.decoder import JSONDecodeError
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobFileManager'):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.buf = b''
self.transport: asyncio.StreamWriter = None
self.writer: typing.Optional['HashBlobWriter'] = None
self.client_version: typing.Optional[int] = None
self.descriptor: typing.Optional['StreamDescriptor'] = None
self.sd_blob: typing.Optional['BlobFile'] = None
self.received = []
self.incoming = asyncio.Event(loop=self.loop)
def connection_made(self, transport):
self.transport = transport
def data_received(self, data: bytes):
if self.incoming.is_set():
try:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob: %s", err)
return
try:
request = json.loads(data.decode())
except (ValueError, JSONDecodeError):
return
self.loop.create_task(self.handle_request(request))
def send_response(self, response: typing.Dict):
self.transport.write(json.dumps(response).encode())
async def handle_request(self, request: typing.Dict):
if self.client_version is None:
if 'version' not in request:
self.transport.close()
return
self.client_version = request['version']
self.send_response({'version': 1})
return
if not self.sd_blob:
if 'sd_blob_hash' not in request:
self.transport.close()
return
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
if not self.sd_blob.get_is_verified():
self.writer = self.sd_blob.open_for_writing()
self.incoming.set()
self.send_response({"send_sd_blob": True})
try:
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_sd_blob": True})
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.incoming.clear()
self.writer.close_handle()
self.writer = None
except (asyncio.TimeoutError, asyncio.CancelledError):
self.send_response({"received_sd_blob": False})
self.incoming.clear()
self.writer.close_handle()
self.writer = None
self.transport.close()
return
else:
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob
)
self.send_response({"send_sd_blob": False, 'needed': [
blob.blob_hash for blob in self.descriptor.blobs[:-1]
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
]})
return
elif self.descriptor:
if 'blob_hash' not in request:
self.transport.close()
return
if request['blob_hash'] not in map(lambda b: b.blob_hash, self.descriptor.blobs[:-1]):
self.send_response({"send_blob": False})
return
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
if not blob.get_is_verified():
self.writer = blob.open_for_writing()
self.incoming.set()
self.send_response({"send_blob": True})
try:
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
self.send_response({"received_blob": True})
except (asyncio.TimeoutError, asyncio.CancelledError):
self.send_response({"received_blob": False})
self.incoming.clear()
self.writer.close_handle()
self.writer = None
else:
self.send_response({"send_blob": False})
return
else:
self.transport.close()
class ReflectorServer:
def __init__(self, blob_manager: 'BlobFileManager'):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
raise Exception("already running")
async def _start_server():
server = await self.loop.create_server(
lambda: ReflectorServerProtocol(self.blob_manager),
interface, port
)
self.started_listening.set()
log.info("Reflector server listening on TCP %s:%i", interface, port)
async with server:
await server.serve_forever()
self.server_task = self.loop.create_task(_start_server())
def stop_server(self):
if self.server_task:
self.server_task.cancel()
self.server_task = None
log.info("Stopped reflector server")
+24 -3
View File
@@ -3,6 +3,7 @@ import asyncio
import typing
import binascii
import logging
import random
from lbrynet.stream.downloader import StreamDownloader
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.schema.claim import ClaimDict
@@ -46,7 +47,8 @@ comparison_operators = {
class StreamManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None):
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
self.loop = loop
self.blob_manager = blob_manager
self.wallet = wallet
@@ -59,6 +61,7 @@ class StreamManager:
self.resume_downloading_task: asyncio.Task = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.fixed_peers = fixed_peers
self.reflector_servers = reflector_servers
async def load_streams_from_database(self):
infos = await self.storage.get_all_lbry_files()
@@ -93,6 +96,20 @@ class StreamManager:
if resumed:
log.info("resuming %i downloads", resumed)
async def reflect_streams(self):
streams = list(self.streams)
batch = []
while streams:
stream = streams.pop()
if not stream.fully_reflected.is_set():
host, port = random.choice(self.reflector_servers)
batch.append(stream.upload_to_reflector(host, port))
if len(batch) >= 10:
await asyncio.gather(*batch)
batch = []
if batch:
await asyncio.gather(*batch)
async def start(self):
await self.load_streams_from_database()
self.resume_downloading_task = self.loop.create_task(self.resume())
@@ -106,9 +123,13 @@ class StreamManager:
while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel()
async def create_stream(self, file_path: str) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path)
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
self.streams.add(stream)
if self.reflector_servers:
host, port = random.choice(self.reflector_servers)
self.loop.create_task(stream.upload_to_reflector(host, port))
return stream
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
-1
View File
@@ -37,6 +37,5 @@ setup(
'torba',
'pyyaml',
'docopt',
'colorama==0.3.7',
],
)
+69 -3
View File
@@ -42,7 +42,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
class TestBlobExchange(BlobExchangeTestBase):
async def _test_transfer_blob(self, blob_hash: str, blob_bytes: bytes):
async def _add_blob_to_server(self, blob_hash: str, blob_bytes: bytes):
# add the blob on the server
server_blob = self.server_blob_manager.get_blob(blob_hash, len(blob_bytes))
writer = server_blob.open_for_writing()
@@ -51,6 +51,7 @@ class TestBlobExchange(BlobExchangeTestBase):
self.assertTrue(os.path.isfile(server_blob.file_path))
self.assertEqual(server_blob.get_is_verified(), True)
async def _test_transfer_blob(self, blob_hash: str):
client_blob = self.client_blob_manager.get_blob(blob_hash)
protocol = BlobExchangeClientProtocol(self.loop, 2)
@@ -64,9 +65,74 @@ class TestBlobExchange(BlobExchangeTestBase):
async def test_transfer_sd_blob(self):
sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02"
mock_sd_blob_bytes = b"""{"blobs": [{"blob_hash": "6f53c72de100f6f007aa1b9720632e2d049cc6049e609ad790b556dba262159f739d5a14648d5701afc84b991254206a", "blob_num": 0, "iv": "3b6110c2d8e742bff66e4314863dee7e", "length": 2097152}, {"blob_hash": "18493bc7c5164b00596153859a0faffa45765e47a6c3f12198a4f7be4658111505b7f8a15ed0162306a0672c4a9b505d", "blob_num": 1, "iv": "df973fa64e73b4ff2677d682cdc32d3e", "length": 2097152}, {"blob_num": 2, "iv": "660d2dc2645da7c7d4540a466fcb0c60", "length": 0}], "key": "6465616462656566646561646265656664656164626565666465616462656566", "stream_hash": "22423c6786584974bd6b462af47ecb03e471da0ef372fe85a4e71a78bef7560c4afb0835c689f03916105404653b7bdf", "stream_name": "746573745f66696c65", "stream_type": "lbryfile", "suggested_file_name": "746573745f66696c65"}"""
return await self._test_transfer_blob(sd_hash, mock_sd_blob_bytes)
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
return await self._test_transfer_blob(sd_hash)
async def test_transfer_blob(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
return await self._test_transfer_blob(blob_hash, mock_blob_bytes)
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
return await self._test_transfer_blob(blob_hash)
async def test_host_same_blob_to_multiple_peers_at_once(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
await second_client_storage.open()
await second_client_blob_manager.setup()
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
protocol = BlobExchangeClientProtocol(self.loop, 2)
# download the blob
await asyncio.gather(
request_blob(
self.loop, second_client_blob, protocol, server_from_second_client.address,
server_from_second_client.tcp_port, 2
),
self._test_transfer_blob(blob_hash)
)
await protocol.close()
self.assertEqual(second_client_blob.get_is_verified(), True)
async def test_host_different_blobs_to_multiple_peers_at_once(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02"
mock_sd_blob_bytes = b"""{"blobs": [{"blob_hash": "6f53c72de100f6f007aa1b9720632e2d049cc6049e609ad790b556dba262159f739d5a14648d5701afc84b991254206a", "blob_num": 0, "iv": "3b6110c2d8e742bff66e4314863dee7e", "length": 2097152}, {"blob_hash": "18493bc7c5164b00596153859a0faffa45765e47a6c3f12198a4f7be4658111505b7f8a15ed0162306a0672c4a9b505d", "blob_num": 1, "iv": "df973fa64e73b4ff2677d682cdc32d3e", "length": 2097152}, {"blob_num": 2, "iv": "660d2dc2645da7c7d4540a466fcb0c60", "length": 0}], "key": "6465616462656566646561646265656664656164626565666465616462656566", "stream_hash": "22423c6786584974bd6b462af47ecb03e471da0ef372fe85a4e71a78bef7560c4afb0835c689f03916105404653b7bdf", "stream_name": "746573745f66696c65", "stream_type": "lbryfile", "suggested_file_name": "746573745f66696c65"}"""
second_client_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, second_client_dir)
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
await second_client_storage.open()
await second_client_blob_manager.setup()
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
protocol = BlobExchangeClientProtocol(self.loop, 2)
await asyncio.gather(
request_blob(
self.loop, second_client_blob, protocol, server_from_second_client.address,
server_from_second_client.tcp_port, 2
),
self._test_transfer_blob(sd_hash)
)
await protocol.close()
self.assertEqual(second_client_blob.get_is_verified(), True)
+4 -4
View File
@@ -49,8 +49,8 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
async def test_one_stopped_first(self):
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
fast_gen = MockAsyncGen(self.loop, 1, 0.1, 5)
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
fast_gen = MockAsyncGen(self.loop, 1, 0.101, 5)
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
await self._test_junction(expected_order, fast_gen, slow_gen)
self.assertEqual(fast_gen.called_close, True)
self.assertEqual(slow_gen.called_close, True)
@@ -62,10 +62,10 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
for i in range(10):
if i == 5:
return
await asyncio.sleep(0.1)
await asyncio.sleep(0.101)
yield 1
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
await self._test_junction(expected_order, fast_gen(), slow_gen)
self.assertEqual(slow_gen.called_close, True)
+2 -2
View File
@@ -48,8 +48,8 @@ class TestStreamDownloader(BlobExchangeTestBase):
async def test_transfer_stream(self):
await self._test_transfer_stream(10)
async def test_transfer_hundred_blob_stream(self):
await self._test_transfer_stream(100)
# async def test_transfer_hundred_blob_stream(self):
# await self._test_transfer_stream(100)
async def test_transfer_stream_bad_first_peer_good_second(self):
await self.setup_stream(2)
+62
View File
@@ -0,0 +1,62 @@
import os
import asyncio
import tempfile
import shutil
from torba.testcase import AsyncioTestCase
from lbrynet.conf import Config
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.stream.reflector.server import ReflectorServer
class TestStreamAssembler(AsyncioTestCase):
async def asyncSetUp(self):
self.loop = asyncio.get_event_loop()
self.key = b'deadbeef' * 4
self.cleartext = os.urandom(20000000)
tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
await self.storage.open()
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
self.stream_manager = StreamManager(self.loop, self.blob_manager, None, self.storage, None, 3.0, 3.0)
server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
self.server_storage = SQLiteStorage(Config(), os.path.join(server_tmp_dir, "lbrynet.sqlite"))
await self.server_storage.open()
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
download_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(download_dir))
# create the stream
file_path = os.path.join(tmp_dir, "test_file")
with open(file_path, 'wb') as f:
f.write(self.cleartext)
self.stream = await self.stream_manager.create_stream(file_path)
async def test_reflect_stream(self):
reflector = ReflectorServer(self.server_blob_manager)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertSetEqual(
set(sent),
set(map(lambda b: b.blob_hash,
self.stream.descriptor.blobs[:-1] + [self.blob_manager.get_blob(self.stream.sd_hash)]))
)
server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash)
self.assertTrue(server_sd_blob.get_is_verified())
self.assertEqual(server_sd_blob.length, server_sd_blob.length)
for blob in self.stream.descriptor.blobs[:-1]:
server_blob = self.server_blob_manager.get_blob(blob.blob_hash)
self.assertTrue(server_blob.get_is_verified())
self.assertEqual(server_blob.length, blob.length)
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertListEqual(sent, [])
+14 -33
View File
@@ -55,48 +55,29 @@ class CLITest(AsyncioTestCase):
self.assertEqual(3, normalize_value('3', key="some_other_thing"))
def test_help(self):
self.assertIn(
'Usage: lbrynet [-v] [--api HOST:PORT]', self.shell(['--help'])
)
self.assertIn('lbrynet [-v] [--api HOST:PORT]', self.shell(['--help']))
# start is special command, with separate help handling
self.assertIn(
'--share-usage-data', self.shell(['start', '--help'])
)
self.assertIn('--share-usage-data', self.shell(['start', '--help']))
# publish is ungrouped command, returns usage only implicitly
self.assertIn(
'publish (<name> | --name=<name>)', self.shell(['publish'])
)
self.assertIn('publish (<name> | --name=<name>)', self.shell(['publish']))
# publish is ungrouped command, with explicit --help
self.assertIn(
'Make a new name claim and publish', self.shell(['publish', '--help'])
)
self.assertIn('Make a new name claim and publish', self.shell(['publish', '--help']))
# account is a group, returns help implicitly
self.assertIn(
'Return the balance of an account',
self.shell(['account'])
)
self.assertIn('Return the balance of an account', self.shell(['account']))
# account is a group, with explicit --help
self.assertIn(
'Return the balance of an account',
self.shell(['account', '--help'])
)
self.assertIn('Return the balance of an account', self.shell(['account', '--help']))
# account add is a grouped command, returns usage implicitly
self.assertIn(
'account_add (<account_name> | --account_name=<account_name>)',
self.shell(['account', 'add'])
)
self.assertIn('account_add (<account_name> | --account_name=<account_name>)', self.shell(['account', 'add']))
# account add is a grouped command, with explicit --help
self.assertIn(
'Add a previously created account from a seed,', self.shell(['account', 'add', '--help'])
)
self.assertIn('Add a previously created account from a seed,', self.shell(['account', 'add', '--help']))
def test_help_error_handling(self):
# person tries `help` command, then they get help even though that's invalid command
self.assertIn('--config FILE', self.shell(['help']))
# help for invalid command, with explicit --help
self.assertIn(
"invalid choice: 'publish1'", self.shell(['publish1', '--help'])
)
self.assertIn('--config FILE', self.shell(['nonexistant', '--help']))
# help for invalid command, implicit
self.assertIn(
"invalid choice: 'publish1'", self.shell(['publish1'])
)
self.assertIn('--config FILE', self.shell(['nonexistant']))
def test_version_command(self):
self.assertEqual(
+25 -2
View File
@@ -46,7 +46,7 @@ class ConfigurationTests(unittest.TestCase):
def test_arguments(self):
parser = argparse.ArgumentParser()
TestConfig.contribute_args(parser)
TestConfig.contribute_to_argparse(parser)
args = parser.parse_args([])
c = TestConfig.create_from_arguments(args)
@@ -180,7 +180,7 @@ class ConfigurationTests(unittest.TestCase):
)
self.assertEqual(c.servers, [('localhost', 5566)])
def test_max_key_fee(self):
def test_max_key_fee_from_yaml(self):
with tempfile.TemporaryDirectory() as temp_dir:
config = os.path.join(temp_dir, 'settings.yml')
with open(config, 'w') as fd:
@@ -196,3 +196,26 @@ class ConfigurationTests(unittest.TestCase):
c.max_key_fee = {'currency': 'BTC', 'amount': 1}
with open(config, 'r') as fd:
self.assertEqual(fd.read(), 'max_key_fee:\n amount: 1\n currency: BTC\n')
with c.update_config():
c.max_key_fee = None
with open(config, 'r') as fd:
self.assertEqual(fd.read(), 'max_key_fee: null\n')
def test_max_key_fee_from_args(self):
parser = argparse.ArgumentParser()
Config.contribute_to_argparse(parser)
# default
args = parser.parse_args([])
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, {'amount': 50.0, 'currency': 'USD'})
# disabled
args = parser.parse_args(['--no-max-key-fee'])
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, None)
# set
args = parser.parse_args(['--max-key-fee', '1.0', 'BTC'])
c = Config.create_from_arguments(args)
self.assertEqual(c.max_key_fee, {'amount': 1.0, 'currency': 'BTC'})