Compare commits
71
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
71fc94b1de | ||
|
|
927876e0c0 | ||
|
|
73893d2f6b | ||
|
|
7afc4c418a | ||
|
|
db7e0b59e4 | ||
|
|
1236c13a00 | ||
|
|
a35385a5c0 | ||
|
|
60a3d11df2 | ||
|
|
cd7c2961dc | ||
|
|
03b1287359 | ||
|
|
131bcc8c9d | ||
|
|
29cc82db45 | ||
|
|
be118de19a | ||
|
|
bf8cb69987 | ||
|
|
37d177178f | ||
|
|
f05b5973ae | ||
|
|
4a3c2e6504 | ||
|
|
8010681915 | ||
|
|
6fe70b58ce | ||
|
|
51ec0a92f7 | ||
|
|
d3a8722ea8 | ||
|
|
ab08f6b35e | ||
|
|
76e3d8861c | ||
|
|
b16356b927 | ||
|
|
1bbedef565 | ||
|
|
e5f049e5ed | ||
|
|
9dee2eee5f | ||
|
|
785471745c | ||
|
|
7afebb02f4 | ||
|
|
8932d90a9e | ||
|
|
96fdb05689 | ||
|
|
9149e371ed | ||
|
|
12ab16f50e | ||
|
|
c185b49ede | ||
|
|
d05eee35e8 | ||
|
|
4df4136c5d | ||
|
|
b7cdd9f2a0 | ||
|
|
ffe828b1d9 | ||
|
|
71bd612c4a | ||
|
|
f176db058e | ||
|
|
b434864f18 | ||
|
|
4da4ab1995 | ||
|
|
920a9b09dc | ||
|
|
8b98a9a4e9 | ||
|
|
476eae3e93 | ||
|
|
6575fb9534 | ||
|
|
1583082acc | ||
|
|
f71d6a4263 | ||
|
|
d2b78a7fc3 | ||
|
|
a98288aa80 | ||
|
|
3a0b4232a5 | ||
|
|
5bdbc9e0d6 | ||
|
|
9d4ef899a6 | ||
|
|
d0e8374636 | ||
|
|
d2b26da3e8 | ||
|
|
59faea9815 | ||
|
|
ce6be620a5 | ||
|
|
580f6e20eb | ||
|
|
7d93edf776 | ||
|
|
b3c5b1e88d | ||
|
|
a84c196916 | ||
|
|
b52f47f273 | ||
|
|
e9d37a8c81 | ||
|
|
d96253cd40 | ||
|
|
4a5d310fd3 | ||
|
|
c6e267e970 | ||
|
|
7b5ae24bea | ||
|
|
92037d786a | ||
|
|
494c48875d | ||
|
|
fc0da99894 | ||
|
|
20e96d1233 |
No files matched your search
@@ -1,8 +1,8 @@
|
||||
<!-- This issue tracker is only for technical issues related to Bitcoin Core.
|
||||
<!-- This issue tracker is only for technical issues related to lbrycrd (the LBRY blockchain).
|
||||
|
||||
General bitcoin questions and/or support requests are best directed to the Bitcoin StackExchange at https://bitcoin.stackexchange.com.
|
||||
General questions and/or support requests are best directed to the community chat at https://chat.lbry.org.
|
||||
|
||||
For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/.
|
||||
For reporting security issues, please email security@lbry.com.
|
||||
|
||||
If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! -->
|
||||
|
||||
@@ -13,7 +13,7 @@ If the node is "stuck" during sync or giving "block checksum mismatch" errors, p
|
||||
|
||||
<!--- How reliably can you reproduce the issue, what are the steps to do so? -->
|
||||
|
||||
<!-- What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? -->
|
||||
<!-- What version of lbrycrd are you using, where did you get it (website, self-compiled, etc)? -->
|
||||
|
||||
<!-- What type of machine are you observing the error on (OS/CPU and disk type)? -->
|
||||
|
||||
|
||||
+9
-18
@@ -7,7 +7,6 @@ cache:
|
||||
stages:
|
||||
- build
|
||||
- test
|
||||
- quality
|
||||
|
||||
jobs:
|
||||
include:
|
||||
@@ -15,7 +14,7 @@ jobs:
|
||||
- &build-template
|
||||
stage: build
|
||||
name: linux
|
||||
env: NAME=linux EXT=
|
||||
env: NAME=linux DOCKER_IMAGE=lbry/build_lbrycrd_gcc EXT=
|
||||
os: linux
|
||||
dist: xenial
|
||||
language: minimal
|
||||
@@ -23,8 +22,9 @@ jobs:
|
||||
- docker
|
||||
install:
|
||||
- mkdir -p ${HOME}/ccache
|
||||
- docker pull $DOCKER_BUILD_IMAGE
|
||||
- docker pull $DOCKER_IMAGE
|
||||
script:
|
||||
- echo "build..."
|
||||
- docker run -v "$(pwd):/lbrycrd" -v "${HOME}/ccache:/ccache" -w /lbrycrd -e CCACHE_DIR=/ccache ${DOCKER_IMAGE} packaging/build_${NAME}_64bit.sh
|
||||
before_deploy:
|
||||
- mkdir -p dist
|
||||
@@ -48,11 +48,11 @@ jobs:
|
||||
|
||||
- <<: *build-template
|
||||
name: windows
|
||||
env: NAME=windows EXT=.exe
|
||||
env: NAME=windows DOCKER_IMAGE=lbry/build_lbrycrd EXT=.exe
|
||||
|
||||
- <<: *build-template
|
||||
name: osx
|
||||
env: NAME=darwin EXT=
|
||||
env: NAME=darwin DOCKER_IMAGE=lbry/build_lbrycrd EXT=
|
||||
before_install:
|
||||
- mkdir -p ./depends/SDKs && pushd depends/SDKs && curl -C - ${MAC_OS_SDK} | tar --skip-old-files -xJ && popd
|
||||
|
||||
@@ -63,12 +63,12 @@ jobs:
|
||||
dist: xenial
|
||||
language: minimal
|
||||
git:
|
||||
depth: 3
|
||||
clone: false
|
||||
install:
|
||||
- mkdir -p testrun && cd testrun
|
||||
- curl http://build.lbry.io/lbrycrd/${TRAVIS_BRANCH}/lbrycrd-${NAME}-test.zip -o temp.zip
|
||||
- unzip temp.zip
|
||||
script: ./test_lbrycrd
|
||||
script: TRIEHASH_FUZZER_BLOCKS=1000 ./test_lbrycrd
|
||||
|
||||
- <<: *test-template
|
||||
# os: windows # doesn't support secrets at the moment
|
||||
@@ -78,20 +78,11 @@ jobs:
|
||||
services:
|
||||
- docker
|
||||
script:
|
||||
- docker pull $DOCKER_WINE_IMAGE
|
||||
- docker run -v "$(pwd):/test" -e "WINEDEBUG=-all" -it $DOCKER_WINE_IMAGE wine "/test/test_lbrycrd.exe"
|
||||
- docker pull lbry/wine
|
||||
- docker run -v "$(pwd):/test" -e "WINEDEBUG=-all" -e "TRIEHASH_FUZZER_BLOCKS=1000" -it lbry/wine wine "/test/test_lbrycrd.exe"
|
||||
|
||||
- <<: *test-template
|
||||
os: osx
|
||||
osx_image: xcode8.3
|
||||
env: NAME=darwin
|
||||
|
||||
|
||||
- stage: quality
|
||||
name: "check format"
|
||||
os: linux
|
||||
dist: xenial
|
||||
language: minimal
|
||||
install:
|
||||
- sudo apt-get install -y clang-format-3.9
|
||||
script: git diff -U0 origin/master -- '*.h' '*.cpp' | ./contrib/devtools/clang-format-diff.py -p1
|
||||
@@ -1,14 +1,35 @@
|
||||
# LBRYcrd - The LBRY blockchain
|
||||
|
||||
[](https://travis-ci.org/lbryio/lbrycrd)
|
||||
[](https://github.com/lbryio/lbry-desktop/blob/master/LICENSE)
|
||||
|
||||
LBRYcrd uses a blockchain similar to bitcoin's to implement an index and payment system for content on the LBRY network. It is a fork of bitcoin core. In addition to the libraries used by bitcoin, LBRYcrd also uses icu4c.
|
||||
LBRYcrd uses a blockchain similar to bitcoin's to implement an index and payment system for content on the LBRY network. It is a fork of [bitcoin core](https://github.com/bitcoin/bitcoin). In addition to the libraries used by bitcoin, LBRYcrd also uses [icu4c](https://github.com/unicode-org/icu/tree/master/icu4c).
|
||||
|
||||
Please read the [lbry.tech overview](https://lbry.tech/overview) for a general understanding of the LBRY pieces. From there you could read the [LBRY spec](https://spec.lbry.com/) for specifics on the data in the blockchain.
|
||||
Please read the [lbry.tech overview](https://lbry.tech/overview) for a general understanding of the LBRY pieces. From there you could read the [LBRY spec](https://spec.lbry.com/) for specifics on the data in the blockchain.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Installation](#installation)
|
||||
2. [Usage](#usage)
|
||||
1. [Examples](#examples)
|
||||
2. [Data directory](#data-directory)
|
||||
3. [Running from Source](#running-from-source)
|
||||
1. [Ubuntu with pulled static dependencies](#ubuntu-with-pulled-static-dependencies)
|
||||
2. [Ubuntu with local shared dependencies](#ubuntu-with-local-shared-dependencies)
|
||||
3. [MacOS (cross-compiled)](<#macos-(cross-compiled)>)
|
||||
4. [MacOS with local shared dependencies](#macos-with-local-shared-dependencies)
|
||||
5. [Windows (cross-compiled)](<#windows-(cross-compiled)>)
|
||||
6. [Use with CLion](#use-with-clion)
|
||||
4. [Contributing](#contributing)
|
||||
- [Testnet](#testnet)
|
||||
5. [Mailing List](#mailing-list)
|
||||
6. [License](#license)
|
||||
7. [Security](#security)
|
||||
8. [Contact](#contact)
|
||||
|
||||
## Installation
|
||||
|
||||
Latest binaries are available from https://github.com/lbryio/lbrycrd/releases. There is no installation procedure; the CLI binaries will run as-is and will have any uncommon dependencies statically linked into the binary. The QT GUI is not supported. LBRYcrd is distributed as a collection of executable files; traditional installers are not provided.
|
||||
Latest binaries are available from https://github.com/lbryio/lbrycrd/releases. There is no installation procedure; the CLI binaries will run as-is and will have any uncommon dependencies statically linked into the binary. The QT GUI is not supported. LBRYcrd is distributed as a collection of executable files; traditional installers are not provided.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -16,7 +37,7 @@ The `lbrycrdd` executable will start a LBRYcrd node and connect you to the LBRYc
|
||||
to interact with lbrycrdd through the command line. Command-line help for both executables are available through
|
||||
the "--help" flag (e.g. `lbrycrdd --help`). Examples:
|
||||
|
||||
#### Examples:
|
||||
#### Examples
|
||||
|
||||
Run `./lbrycrdd -server -daemon` to start lbrycrdd in the background.
|
||||
|
||||
@@ -25,7 +46,8 @@ Run `./lbrycrd-cli -getinfo` to check for some basic information about your LBRY
|
||||
Run `./lbrycrd-cli help` to get a list of all commands that you can run. To get help on specific commands run `./lbrycrd-cli [command_name] help`
|
||||
|
||||
Test locally:
|
||||
```
|
||||
|
||||
```sh
|
||||
./lbrycrdd -server -regtest -txindex # run this in its own window
|
||||
./lbrycrd-cli -regtest generate 120 # mine 20 spendable coins
|
||||
./lbrycrd-cli -regtest claimname my_name deadbeef 1 # hold a name claim with 1 coin
|
||||
@@ -35,24 +57,26 @@ Test locally:
|
||||
./lbrycrd-cli -regtest stop # kill lbrycrdd
|
||||
rm -fr ~/.lbrycrd/regtest/ # destroy regtest data
|
||||
```
|
||||
|
||||
For further understanding of a "regtest" setup, see the local stack setup instructions here: https://lbry.tech/resources/regtest-setup
|
||||
|
||||
The CLI help is also browsable online at https://lbry.tech/api/blockchain
|
||||
|
||||
#### Data directory:
|
||||
#### Data directory
|
||||
|
||||
Lbrycrdd will use the below default data directories (changeable with -datadir):
|
||||
|
||||
```
|
||||
```sh
|
||||
Windows: %APPDATA%\lbrycrd
|
||||
Mac: ~/Library/Application Support/lbrycrd
|
||||
Unix: ~/.lbrycrd
|
||||
```
|
||||
|
||||
The data directory contains various things such as your default wallet (wallet.dat), debug logs (debug.log), and blockchain data. You can optionally create a configuration file lbrycrd.conf in the default data directory which will be used by default when running lbrycrdd.
|
||||
|
||||
For a list of configuration parameters, run `./lbrycrdd --help`. Below is a sample lbrycrd.conf to enable JSON RPC server on lbrycrdd.
|
||||
|
||||
```
|
||||
```sh
|
||||
rpcuser=lbry
|
||||
rpcpassword=xyz123456790
|
||||
daemon=1
|
||||
@@ -61,15 +85,19 @@ txindex=1
|
||||
```
|
||||
|
||||
## Running from Source
|
||||
The easiest way to compile is to utilize the Docker image that contains the necessary compilers: lbry/build_lbrycrd. This will allow you to reproduce the build as made on our build servers. I this sample we map a local lbrycrd folder and a local ccache folder inside the image:
|
||||
```
|
||||
|
||||
The easiest way to compile is to utilize the Docker image that contains the necessary compilers: lbry/build_lbrycrd. This will allow you to reproduce the build as made on our build servers. In this sample we map a local lbrycrd folder and a local ccache folder inside the image:
|
||||
```sh
|
||||
git clone https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
docker run -v "$(pwd):/lbrycrd" --rm -v "${HOME}/ccache:/ccache" -w /lbrycrd -e CCACHE_DIR=/ccache lbry/build_lbrycrd packaging/build_linux_64bit.sh
|
||||
```
|
||||
|
||||
Some examples of compiling directly:
|
||||
#### Ubuntu with pulled static dependencies:
|
||||
```
|
||||
|
||||
#### Ubuntu with pulled static dependencies
|
||||
|
||||
```sh
|
||||
sudo apt install build-essential git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates
|
||||
git clone https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
@@ -77,16 +105,20 @@ cd lbrycrd
|
||||
./src/test/test_lbrycrd
|
||||
|
||||
```
|
||||
|
||||
Other Linux distros would be similar. The build shell script is fairly trivial; take a peek at its contents.
|
||||
#### Ubuntu with local shared dependencies:
|
||||
|
||||
#### Ubuntu with local shared dependencies
|
||||
|
||||
Note: using untested dependencies may lead to conflicting results.
|
||||
```
|
||||
|
||||
```sh
|
||||
sudo add-apt-repository ppa:bitcoin/bitcoin
|
||||
sudo apt-get update
|
||||
sudo apt-get install libdb4.8-dev libdb4.8++-dev libicu-dev libssl-dev libevent-dev \
|
||||
sudo apt-get install libdb4.8-dev libdb4.8++-dev libicu-dev libssl-dev libevent-dev \
|
||||
build-essential git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates \
|
||||
libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev
|
||||
|
||||
|
||||
# optionally include libminiupnpc-dev libzmq3-dev
|
||||
|
||||
git clone https://github.com/lbryio/lbrycrd.git
|
||||
@@ -97,11 +129,13 @@ make -j$(nproc)
|
||||
./src/lbrycrdd -server ...
|
||||
|
||||
```
|
||||
#### MacOS (cross-compiled):
|
||||
```
|
||||
|
||||
#### MacOS (cross-compiled)
|
||||
|
||||
```sh
|
||||
sudo apt-get install clang llvm git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates \
|
||||
libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev
|
||||
|
||||
|
||||
git clone https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
# download MacOS SDK from your favorite source
|
||||
@@ -110,9 +144,12 @@ tar ... extract SDK to depends/SDKs/MacOSX10.11.sdk
|
||||
./packaging/build_darwin_64bit.sh
|
||||
|
||||
```
|
||||
|
||||
Look in packaging/build_darwin_64bit.sh for further understanding.
|
||||
#### MacOS with local shared dependencies:
|
||||
```
|
||||
|
||||
#### MacOS with local shared dependencies
|
||||
|
||||
```sh
|
||||
brew install boost berkeley-db@4 icu4c libevent
|
||||
# fix conflict with gawk pulled first:
|
||||
brew reinstall readline
|
||||
@@ -127,14 +164,17 @@ CONFIG_SITE=$(pwd)/depends/x86_64-apple-darwin15.6.0/share/config.site ./configu
|
||||
make -j$(sysctl -n hw.ncpu)
|
||||
|
||||
```
|
||||
#### Windows (cross-compiled):
|
||||
|
||||
#### Windows (cross-compiled)
|
||||
|
||||
Compiling on MS Windows (outside of WSL) is not supported. The Windows build is cross-compiled from Linux like so:
|
||||
```
|
||||
|
||||
```sh
|
||||
sudo apt-get install build-essential git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates \
|
||||
g++-mingw-w64-x86-64 mingw-w64-x86-64-dev
|
||||
|
||||
|
||||
update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
|
||||
|
||||
|
||||
git clone https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
./packaging/build_windows_64bit.sh
|
||||
@@ -143,13 +183,16 @@ cd lbrycrd
|
||||
|
||||
If you encounter any errors, please check `doc/build-*.md` for further instructions. If you're still stuck, [create an issue](https://github.com/lbryio/lbrycrd/issues/new) with the output of that command, your system info, and any other information you think might be helpful. The scripts in the packaging folder are simple and will grant extra light on the build process as needed.
|
||||
|
||||
#### Use with CLion:
|
||||
#### Use with CLion
|
||||
|
||||
CLion has not traditionally supported Autotools projects, although some progress on that is now in the works. We do include a cmake build file for compiling lbrycrd. See contrib/cmake. Alas, CLion doesn't support external projects in cmake, so that particular approach is also insufficient. CLion does support "compile_commands.json" projects. Fortunately, this can be easily generated for lbrycrd like so:
|
||||
```
|
||||
|
||||
```sh
|
||||
pip install --user compiledb
|
||||
./autogen.sh && ./configure --enable-static=no --enable-shared --with-pic --without-gui CXXFLAGS="-O0 -g" CFLAGS="-O0 -g" # or whatever normal lbrycrd config
|
||||
compiledb make -j10
|
||||
```
|
||||
|
||||
Then open the newly generated compile_commands.json file as a project in CLion. Debugging is supported if you compiled with `-g`. To enable that you will need to create a target in CLion by going to File -> Settings -> Build -> Custom Build Targets. Add an empty target with your choice of name. From there you can go to "Edit Configurations", typically found in a drop-down at the top of the editor. Add a Custom Build Application, select your new target, select the compiled file (i.e. test_lbrycrd or lbrycrdd, etc), and then add any necessary command line parameters. Ensure that there is nothing in the "Before launch" section.
|
||||
|
||||
## Contributing
|
||||
@@ -167,8 +210,8 @@ regularly to indicate new official, stable release versions.
|
||||
Testing and code review is the bottleneck for development; we get more pull
|
||||
requests than we can review and test on short notice. Please be patient and help out by testing
|
||||
other people's pull requests, and remember this is a security-critical project where any mistake might cost people
|
||||
lots of money. Developers are strongly encouraged to write [unit tests](/doc/unit-tests.md) for new code and to
|
||||
submit new unit tests for old code. Unit tests are compiled by default and can be run with `src/test/test_lbrycrd`.
|
||||
lots of money. Developers are strongly encouraged to write [unit tests](/src/test/README.md) for new code and to
|
||||
submit new unit tests for old code. Unit tests are compiled by default and can be run with `src/test/test_lbrycrd`
|
||||
|
||||
The Travis CI system makes sure that every pull request is built, and that unit and sanity tests are automatically run. See https://travis-ci.org/lbryio/lbrycrd
|
||||
|
||||
@@ -176,7 +219,11 @@ The Travis CI system makes sure that every pull request is built, and that unit
|
||||
|
||||
Testnet is maintained for testing purposes and can be accessed using the command `./lbrycrdd -testnet`. If you would like to obtain testnet credits, please contact brannon@lbry.com or grin@lbry.com .
|
||||
|
||||
It is easy to solo mine on testnet. (It's easy on mainnet too, but much harder to win.) For instructions see https://github.com/lbryio/sgminer-gm and https://github.com/lbryio/lbrycrd/tree/master/contrib/mining
|
||||
It is easy to solo mine on testnet. (It's easy on mainnet too, but much harder to win.) For instructions see [SGMiner](https://github.com/lbryio/sgminer-gm) and [Mining Contributions](https://github.com/lbryio/lbrycrd/tree/master/contrib/mining)
|
||||
|
||||
## Mailing List
|
||||
|
||||
We maintain a mailing list for notifications of upgrades, security issues, and soft/hard forks. To join, visit [https://lbry.com/forklist](https://lbry.com/forklist).
|
||||
|
||||
## License
|
||||
|
||||
@@ -184,11 +231,9 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues.
|
||||
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
|
||||
We take security seriously. Please contact [security@lbry.com](mailto:security@lbry.com) regarding any security issues.
|
||||
Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@BrannonKing](https://github.com/BrannonKing) (brannon@lbry.com)
|
||||
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ export LC_ALL=C
|
||||
set -e
|
||||
srcdir="$(dirname $0)"
|
||||
cd "$srcdir"
|
||||
if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="`which glibtoolize 2>/dev/null`"; then
|
||||
if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="$(which glibtoolize 2>/dev/null)"; then
|
||||
LIBTOOLIZE="${GLIBTOOLIZE}"
|
||||
export LIBTOOLIZE
|
||||
fi
|
||||
|
||||
+3
-3
@@ -2,10 +2,10 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N)
|
||||
AC_PREREQ([2.60])
|
||||
define(_CLIENT_VERSION_MAJOR, 0)
|
||||
define(_CLIENT_VERSION_MINOR, 17)
|
||||
define(_CLIENT_VERSION_REVISION, 2)
|
||||
define(_CLIENT_VERSION_BUILD, 1)
|
||||
define(_CLIENT_VERSION_REVISION, 3)
|
||||
define(_CLIENT_VERSION_BUILD, 3)
|
||||
define(_CLIENT_VERSION_IS_RELEASE, true)
|
||||
define(_COPYRIGHT_YEAR, 2019)
|
||||
define(_COPYRIGHT_YEAR, 2021)
|
||||
define(_COPYRIGHT_HOLDERS,[The %s developers])
|
||||
define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[LBRYcrd Core]])
|
||||
AC_INIT([LBRYcrd Core],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION],[https://github.com/lbryio/lbrycrd/issues],[lbrycrd],[https://lbry.com/])
|
||||
|
||||
@@ -36,13 +36,13 @@ if [ -z "${CODESIGN_ALLOCATE}" ]; then
|
||||
fi
|
||||
|
||||
find ${TEMPDIR} -name "*.sign" | while read i; do
|
||||
SIZE=`stat -c %s "${i}"`
|
||||
TARGET_FILE="`echo "${i}" | sed 's/\.sign$//'`"
|
||||
SIZE=$(stat -c %s "${i}")
|
||||
TARGET_FILE="$(echo "${i}" | sed 's/\.sign$//')"
|
||||
|
||||
echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}"
|
||||
${CODESIGN_ALLOCATE} -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp"
|
||||
|
||||
OFFSET=`${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
|
||||
OFFSET=$(${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g')
|
||||
if [ -z ${QUIET} ]; then
|
||||
echo "Attaching signature at offset ${OFFSET}"
|
||||
fi
|
||||
|
||||
@@ -27,19 +27,19 @@ ${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}"
|
||||
|
||||
grep -v CodeResources < "${TEMPLIST}" | while read i; do
|
||||
TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`"
|
||||
SIZE=`pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g'`
|
||||
OFFSET=`pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
|
||||
SIZE=$(pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g')
|
||||
OFFSET=$(pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g')
|
||||
SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign"
|
||||
DIRNAME="`dirname "${SIGNFILE}"`"
|
||||
DIRNAME="$(dirname "${SIGNFILE}")"
|
||||
mkdir -p "${DIRNAME}"
|
||||
echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}"
|
||||
dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null
|
||||
done
|
||||
|
||||
grep CodeResources < "${TEMPLIST}" | while read i; do
|
||||
TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`"
|
||||
TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")"
|
||||
RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}"
|
||||
DIRNAME="`dirname "${RESOURCE}"`"
|
||||
DIRNAME="$(dirname "${RESOURCE}")"
|
||||
mkdir -p "${DIRNAME}"
|
||||
echo "Adding resource for: \"${TARGETFILE}\""
|
||||
cp "${i}" "${RESOURCE}"
|
||||
|
||||
@@ -36,7 +36,7 @@ insert into coins(name, symbol, symbol2, algo, enable, auto_ready, rpcuser, rpcp
|
||||
values('Local LBRY Instance', 'LBC', 'LBC', 'lbry', 1, 1, 'ruser', 'rpswd', '127.0.0.1', 19245, 1, 'utf-8', 0, 1, 0, 0, 0);
|
||||
exit
|
||||
```
|
||||
Use port 19245 for testnet, port 9245 for main.
|
||||
Use port 19245 for testnet, port 9245 for main. Set usesegwit to 1 after the segwit fork is enabled on December 11, 2019.
|
||||
#### 3. Run the stratum server:
|
||||
```
|
||||
docker run --network host -d lbry/yiimp_stratum
|
||||
@@ -47,10 +47,11 @@ docker run --network host -it lbry/yiimp_stratum bash
|
||||
cat config/lbry.conf
|
||||
./stratum config/lbry
|
||||
```
|
||||
When testing with an ASIC you may need to modify the TCP server address in said lbry.conf file to be an external IP address.
|
||||
|
||||
#### 4. Connect sgminer to it:
|
||||
```
|
||||
sgminer -k lbry -o stratum+tcp://127.0.0.1:3334/ -D -T -O mn824Su1wX7ip8WcNYzXwwWqvBvkeWGRo6:x
|
||||
```
|
||||
The username there is the account to receive payments from the pool. The password is unused. Tested with https://github.com/lbryio/sgminer-gm.
|
||||
You can use whatever miner you prefer.
|
||||
You can use whatever miner you prefer.
|
||||
@@ -23,7 +23,7 @@ TIMESERVER=http://timestamp.comodoca.com
|
||||
CERTFILE="win-codesign.cert"
|
||||
|
||||
mkdir -p "${OUTSUBDIR}"
|
||||
basename -a `ls -1 "${SRCDIR}"/*-unsigned.exe` | while read UNSIGNED; do
|
||||
basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do
|
||||
echo Signing "${UNSIGNED}"
|
||||
"${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@"
|
||||
"${OSSLSIGNCODE}" extract-signature -pem -in "${WORKDIR}/${UNSIGNED}" -out "${OUTSUBDIR}/${UNSIGNED}.pem" && rm "${WORKDIR}/${UNSIGNED}"
|
||||
|
||||
+1
-1
@@ -131,7 +131,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
|
||||
-e 's|@build_os@|$(build_os)|' \
|
||||
-e 's|@host_os@|$(host_os)|' \
|
||||
-e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \
|
||||
-e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \
|
||||
-e 's|@CXXFLAGS@|$(strip -pipe $(host_$(release_type)_CXXFLAGS))|' \
|
||||
-e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \
|
||||
-e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
|
||||
-e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -7,12 +7,13 @@ darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroo
|
||||
darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B $(host_prefix)/native/bin
|
||||
|
||||
darwin_CFLAGS=-pipe
|
||||
darwin_CXXFLAGS=$(darwin_CFLAGS)
|
||||
darwin_CXXFLAGS=$(darwin_CFLAGS) -std=c++11
|
||||
|
||||
darwin_release_CFLAGS=-O2
|
||||
darwin_release_CFLAGS=-O2 -g
|
||||
darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
|
||||
|
||||
darwin_debug_CFLAGS=-Og
|
||||
darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
|
||||
darwin_debug_CFLAGS=-Og -g
|
||||
darwin_debug_CXXFLAGS=-O0 -g
|
||||
|
||||
darwin_native_toolchain=native_cctools
|
||||
|
||||
darwin_native_toolchain=native_cctools
|
||||
@@ -1,11 +1,14 @@
|
||||
linux_CFLAGS=-pipe
|
||||
linux_CXXFLAGS=$(linux_CFLAGS)
|
||||
linux_CXXFLAGS=$(linux_CFLAGS) -std=c++11
|
||||
|
||||
linux_release_CFLAGS=-O2
|
||||
linux_release_CFLAGS=-O3 -g
|
||||
ifeq (1,$(shell ldd --version | head -1 | awk '{print $$NF < 2.28}'))
|
||||
linux_release_CFLAGS+= -include $(BASEDIR)/glibc_version_header/force_link_glibc_2.19.h
|
||||
endif
|
||||
linux_release_CXXFLAGS=$(linux_release_CFLAGS)
|
||||
|
||||
linux_debug_CFLAGS=-Og
|
||||
linux_debug_CXXFLAGS=$(linux_debug_CFLAGS)
|
||||
linux_debug_CFLAGS=-O1 -g
|
||||
linux_debug_CXXFLAGS=-O0 -g
|
||||
|
||||
linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
mingw32_CFLAGS=-pipe
|
||||
mingw32_CXXFLAGS=$(mingw32_CFLAGS)
|
||||
mingw32_CXXFLAGS=$(mingw32_CFLAGS) -std=c++11
|
||||
|
||||
mingw32_release_CFLAGS=-O2
|
||||
mingw32_release_CFLAGS=-O2 -g
|
||||
mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS)
|
||||
|
||||
mingw32_debug_CFLAGS=-O1
|
||||
mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS)
|
||||
mingw32_debug_CFLAGS=-O1 -g
|
||||
mingw32_debug_CXXFLAGS=-O0 -g
|
||||
|
||||
mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package=bdb
|
||||
$(package)_version=4.8.30
|
||||
$(package)_download_path=http://download.oracle.com/berkeley-db
|
||||
$(package)_download_path=https://download.oracle.com/berkeley-db
|
||||
$(package)_file_name=db-$($(package)_version).NC.tar.gz
|
||||
$(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
|
||||
$(package)_build_subdir=build_unix
|
||||
@@ -9,7 +9,7 @@ define $(package)_set_vars
|
||||
$(package)_config_opts=--disable-shared --enable-cxx --disable-replication
|
||||
$(package)_config_opts_mingw32=--enable-mingw
|
||||
$(package)_config_opts_linux=--with-pic
|
||||
$(package)_cxxflags=-std=c++11
|
||||
$(package)_cppflags_mingw32=-DUNICODE -D_UNICODE
|
||||
endef
|
||||
|
||||
define $(package)_preprocess_cmds
|
||||
@@ -29,3 +29,4 @@ endef
|
||||
define $(package)_stage_cmds
|
||||
$(MAKE) DESTDIR=$($(package)_staging_dir) install_lib install_include
|
||||
endef
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package=boost
|
||||
$(package)_version=1_69_0
|
||||
$(package)_download_path=https://dl.bintray.com/boostorg/release/1.69.0/source/
|
||||
$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/
|
||||
$(package)_file_name=$(package)_$($(package)_version).tar.bz2
|
||||
$(package)_sha256_hash=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406
|
||||
$(package)_dependencies=icu
|
||||
|
||||
@@ -22,7 +22,6 @@ define $(package)_preprocess_cmds
|
||||
PKG_CONFIG_SYSROOT_DIR=/ \
|
||||
PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig \
|
||||
PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig \
|
||||
sed -i.old 's/^GEN_DEPS.cc.*/& $(CXXFLAGS)/' source/config/mh-mingw* && \
|
||||
mkdir -p build && cd build && \
|
||||
../source/runConfigureICU Linux $($(package)_standard_opts) CXXFLAGS=-std=c++11 && \
|
||||
$(MAKE) && cd ..
|
||||
@@ -32,6 +31,8 @@ define $(package)_config_cmds
|
||||
PKG_CONFIG_SYSROOT_DIR=/ \
|
||||
PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig \
|
||||
PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig \
|
||||
sed -i.old 's|^GEN_DEPS.c=.*|& $($(package)_cflags)|' config/mh-mingw* && \
|
||||
sed -i.old 's|^GEN_DEPS.cc=.*|& $($(package)_cxxflags)|' config/mh-mingw* && \
|
||||
$($(package)_autoconf)
|
||||
endef
|
||||
|
||||
|
||||
@@ -27,4 +27,5 @@ define $(package)_stage_cmds
|
||||
endef
|
||||
|
||||
define $(package)_postprocess_cmds
|
||||
rm lib/*.la
|
||||
endef
|
||||
@@ -1,6 +1,6 @@
|
||||
package=miniupnpc
|
||||
$(package)_version=2.0.20180203
|
||||
$(package)_download_path=http://miniupnp.free.fr/files
|
||||
$(package)_download_path=https://miniupnp.tuxfamily.org/files/
|
||||
$(package)_file_name=$(package)-$($(package)_version).tar.gz
|
||||
$(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ $(package)_file_name=$(package)-$($(package)_version).tar.gz
|
||||
$(package)_sha256_hash=8f9faeaebad088e772f4ef5e38252d472be4d878c6b3a2718c10a4fcebe7a41c
|
||||
|
||||
define $(package)_set_vars
|
||||
$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
|
||||
$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc) $($(package)_cflags) $($(package)_cppflags)"
|
||||
$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
|
||||
$(package)_config_opts+=no-camellia
|
||||
$(package)_config_opts+=no-capieng
|
||||
@@ -42,7 +42,6 @@ $(package)_config_opts+=no-weak-ssl-ciphers
|
||||
$(package)_config_opts+=no-whirlpool
|
||||
$(package)_config_opts+=no-zlib
|
||||
$(package)_config_opts+=no-zlib-dynamic
|
||||
$(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags)
|
||||
$(package)_config_opts_linux=-fPIC -Wa,--noexecstack
|
||||
$(package)_config_opts_x86_64_linux=linux-x86_64
|
||||
$(package)_config_opts_i686_linux=linux-generic32
|
||||
|
||||
@@ -4,7 +4,6 @@ $(package)_download_path=$(native_$(package)_download_path)
|
||||
$(package)_file_name=$(native_$(package)_file_name)
|
||||
$(package)_sha256_hash=$(native_$(package)_sha256_hash)
|
||||
$(package)_dependencies=native_$(package)
|
||||
$(package)_cxxflags=-std=c++11
|
||||
|
||||
define $(package)_set_vars
|
||||
$(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc
|
||||
|
||||
@@ -6,9 +6,10 @@ $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d83
|
||||
$(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch
|
||||
|
||||
define $(package)_set_vars
|
||||
$(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror
|
||||
$(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror --disable-drafts
|
||||
$(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci
|
||||
$(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov
|
||||
$(package)_config_opts_linux=--with-pic
|
||||
$(package)_cxxflags=-std=c++11
|
||||
endef
|
||||
|
||||
define $(package)_preprocess_cmds
|
||||
@@ -31,5 +32,5 @@ endef
|
||||
|
||||
define $(package)_postprocess_cmds
|
||||
sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \
|
||||
rm -rf bin share
|
||||
rm -rf bin share lib/*.la
|
||||
endef
|
||||
@@ -30,13 +30,13 @@ if which ccache >/dev/null; then
|
||||
fi
|
||||
|
||||
pushd depends
|
||||
make -j`getconf _NPROCESSORS_ONLN` HOST=x86_64-apple-darwin14 NO_QT=1 V=1
|
||||
make -j$(getconf _NPROCESSORS_ONLN) HOST=x86_64-apple-darwin14 NO_QT=1 V=1
|
||||
popd
|
||||
|
||||
./autogen.sh
|
||||
DEPS_DIR=`pwd`/depends/x86_64-apple-darwin14
|
||||
DEPS_DIR=$(pwd)/depends/x86_64-apple-darwin14
|
||||
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --enable-reduce-exports --without-gui --with-icu="${DEPS_DIR}" --enable-static --disable-shared
|
||||
make -j`getconf _NPROCESSORS_ONLN`
|
||||
make -j$(getconf _NPROCESSORS_ONLN)
|
||||
${DEPS_DIR}/native/bin/x86_64-apple-darwin14-strip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx
|
||||
|
||||
if which ccache >/dev/null; then
|
||||
|
||||
@@ -20,13 +20,13 @@ export CXXFLAGS="${CXXFLAGS:--frecord-gcc-switches}"
|
||||
echo "CXXFLAGS set to $CXXFLAGS"
|
||||
|
||||
cd depends
|
||||
make -j`getconf _NPROCESSORS_ONLN` HOST=x86_64-pc-linux-gnu NO_QT=1 V=1
|
||||
make -j$(getconf _NPROCESSORS_ONLN) HOST=x86_64-pc-linux-gnu NO_QT=1 V=1
|
||||
cd ..
|
||||
|
||||
./autogen.sh
|
||||
DEPS_DIR=`pwd`/depends/x86_64-pc-linux-gnu
|
||||
DEPS_DIR=$(pwd)/depends/x86_64-pc-linux-gnu
|
||||
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --enable-static --disable-shared --with-pic --without-gui
|
||||
make -j`getconf _NPROCESSORS_ONLN`
|
||||
make -j$(getconf _NPROCESSORS_ONLN)
|
||||
strip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx
|
||||
|
||||
if which ccache >/dev/null; then
|
||||
@@ -34,4 +34,4 @@ if which ccache >/dev/null; then
|
||||
ccache -s
|
||||
fi
|
||||
|
||||
echo "Linux 64bit build is complete"
|
||||
echo "Linux 64bit build is complete"
|
||||
@@ -20,17 +20,14 @@ if which ccache >/dev/null; then
|
||||
ccache -ps
|
||||
fi
|
||||
|
||||
export CXXFLAGS="${CXXFLAGS:--frecord-gcc-switches}"
|
||||
echo "CXXFLAGS set to $CXXFLAGS"
|
||||
|
||||
pushd depends
|
||||
make -j`getconf _NPROCESSORS_ONLN` HOST=i686-w64-mingw32 NO_QT=1 V=1
|
||||
make -j$(getconf _NPROCESSORS_ONLN) HOST=i686-w64-mingw32 NO_QT=1 V=1
|
||||
popd
|
||||
|
||||
./autogen.sh
|
||||
DEPS_DIR=`pwd`/depends/i686-w64-mingw32
|
||||
DEPS_DIR=$(pwd)/depends/i686-w64-mingw32
|
||||
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --prefix=/ --without-gui --with-icu="$DEPS_DIR" --enable-static --disable-shared
|
||||
make -j`getconf _NPROCESSORS_ONLN`
|
||||
make -j$(getconf _NPROCESSORS_ONLN)
|
||||
i686-w64-mingw32-strip src/lbrycrdd.exe src/lbrycrd-cli.exe src/lbrycrd-tx.exe
|
||||
|
||||
if which ccache >/dev/null; then
|
||||
|
||||
@@ -19,17 +19,14 @@ if which ccache >/dev/null; then
|
||||
ccache -ps
|
||||
fi
|
||||
|
||||
export CXXFLAGS="${CXXFLAGS:--frecord-gcc-switches}"
|
||||
echo "CXXFLAGS set to $CXXFLAGS"
|
||||
|
||||
pushd depends
|
||||
make -j`getconf _NPROCESSORS_ONLN` HOST=x86_64-w64-mingw32 NO_QT=1 V=1
|
||||
make -j$(getconf _NPROCESSORS_ONLN) HOST=x86_64-w64-mingw32 NO_QT=1 V=1
|
||||
popd
|
||||
|
||||
./autogen.sh
|
||||
DEPS_DIR=`pwd`/depends/x86_64-w64-mingw32
|
||||
DEPS_DIR=$(pwd)/depends/x86_64-w64-mingw32
|
||||
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --prefix=/ --without-gui --with-icu="$DEPS_DIR" --enable-static --disable-shared
|
||||
make -j`getconf _NPROCESSORS_ONLN`
|
||||
make -j$(getconf _NPROCESSORS_ONLN)
|
||||
x86_64-w64-mingw32-strip src/lbrycrdd.exe src/lbrycrd-cli.exe src/lbrycrd-tx.exe
|
||||
|
||||
if which ccache >/dev/null; then
|
||||
|
||||
+3
-1
@@ -155,6 +155,7 @@ BITCOIN_CORE_H = \
|
||||
reverse_iterator.h \
|
||||
reverselock.h \
|
||||
rpc/blockchain.h \
|
||||
rpc/claimrpchelp.h \
|
||||
rpc/client.h \
|
||||
rpc/mining.h \
|
||||
rpc/protocol.h \
|
||||
@@ -475,6 +476,7 @@ lbrycrdd_LDADD = \
|
||||
$(LIBBITCOIN_CONSENSUS) \
|
||||
$(LIBBITCOIN_CRYPTO) \
|
||||
$(LIBLEVELDB) \
|
||||
$(LIBLEVELDB_SSE42) \
|
||||
$(LIBMEMENV) \
|
||||
$(LIBSECP256K1)
|
||||
|
||||
@@ -572,7 +574,7 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps)
|
||||
clean-local:
|
||||
-$(MAKE) -C secp256k1 clean
|
||||
-$(MAKE) -C univalue clean
|
||||
-$(MAKE) -C leveldb clean
|
||||
-rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno
|
||||
-rm -f config.h
|
||||
-rm -rf test/__pycache__
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ bench_bench_bitcoin_LDADD = \
|
||||
$(LIBBITCOIN_CONSENSUS) \
|
||||
$(LIBBITCOIN_CRYPTO) \
|
||||
$(LIBLEVELDB) \
|
||||
$(LIBLEVELDB_SSE42) \
|
||||
$(LIBMEMENV) \
|
||||
$(LIBSECP256K1) \
|
||||
$(LIBUNIVALUE)
|
||||
|
||||
@@ -2,23 +2,148 @@
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
SUBDIRS = leveldb
|
||||
|
||||
LIBLEVELDB_INT = leveldb/libleveldb.a
|
||||
LIBMEMENV_INT = leveldb/libmemenv.a
|
||||
LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a
|
||||
|
||||
EXTRA_LIBRARIES += $(LIBLEVELDB_INT)
|
||||
EXTRA_LIBRARIES += $(LIBMEMENV_INT)
|
||||
EXTRA_LIBRARIES += $(LIBLEVELDB_SSE42_INT)
|
||||
|
||||
LIBLEVELDB += $(LIBLEVELDB_INT)
|
||||
LIBMEMENV += $(LIBMEMENV_INT)
|
||||
LIBLEVELDB_SSE42 = $(LIBLEVELDB_SSE42_INT)
|
||||
|
||||
LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include
|
||||
LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv
|
||||
LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb
|
||||
|
||||
leveldb/libleveldb.a:
|
||||
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C leveldb
|
||||
LEVELDB_CPPFLAGS_INT =
|
||||
LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb
|
||||
LEVELDB_CPPFLAGS_INT += $(LEVELDB_TARGET_FLAGS)
|
||||
LEVELDB_CPPFLAGS_INT += -DLEVELDB_ATOMIC_PRESENT
|
||||
LEVELDB_CPPFLAGS_INT += -D__STDC_LIMIT_MACROS
|
||||
|
||||
leveldb/libmemenv.a: leveldb/libleveldb.a
|
||||
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C leveldb memenv_test
|
||||
if TARGET_WINDOWS
|
||||
LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1
|
||||
else
|
||||
LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX
|
||||
endif
|
||||
|
||||
leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS)
|
||||
leveldb_libleveldb_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
|
||||
|
||||
leveldb_libleveldb_a_SOURCES=
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/atomic_pointer.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/port_example.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/win/stdint.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/port.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/block.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/format.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/random.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h
|
||||
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc
|
||||
|
||||
if TARGET_WINDOWS
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/util/env_win.cc
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.cc
|
||||
else
|
||||
leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.cc
|
||||
endif
|
||||
|
||||
leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
|
||||
leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
|
||||
leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc
|
||||
leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h
|
||||
|
||||
leveldb_libleveldb_sse42_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
|
||||
leveldb_libleveldb_sse42_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
|
||||
if ENABLE_HWCRC32
|
||||
leveldb_libleveldb_sse42_a_CPPFLAGS += -DLEVELDB_PLATFORM_POSIX_SSE
|
||||
leveldb_libleveldb_sse42_a_CXXFLAGS += $(SSE42_CXXFLAGS)
|
||||
endif
|
||||
leveldb_libleveldb_sse42_a_SOURCES = leveldb/port/port_posix_sse.cc
|
||||
@@ -408,7 +408,7 @@ endif
|
||||
if ENABLE_ZMQ
|
||||
qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
|
||||
endif
|
||||
qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \
|
||||
qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
|
||||
$(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
|
||||
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
|
||||
qt_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
|
||||
|
||||
@@ -63,7 +63,7 @@ if ENABLE_ZMQ
|
||||
qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
|
||||
endif
|
||||
qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \
|
||||
$(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
|
||||
$(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
|
||||
$(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
|
||||
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
|
||||
qt_test_test_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
|
||||
|
||||
@@ -66,6 +66,11 @@ BITCOIN_TESTS =\
|
||||
test/net_tests.cpp \
|
||||
test/claimtriecache_tests.cpp \
|
||||
test/claimtriebranching_tests.cpp \
|
||||
test/claimtrieexpirationfork_tests.cpp \
|
||||
test/claimtriefixture.cpp \
|
||||
test/claimtriehashfork_tests.cpp \
|
||||
test/claimtrienormalization_tests.cpp \
|
||||
test/claimtrierpc_tests.cpp \
|
||||
test/nameclaim_tests.cpp \
|
||||
test/netbase_tests.cpp \
|
||||
test/pmt_tests.cpp \
|
||||
@@ -122,7 +127,7 @@ test_test_lbrycrd_LDADD += $(LIBBITCOIN_WALLET)
|
||||
endif
|
||||
|
||||
test_test_lbrycrd_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
|
||||
$(LIBLEVELDB) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
|
||||
$(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
|
||||
test_test_lbrycrd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
|
||||
|
||||
test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include <bloom.h>
|
||||
#include <nameclaim.h>
|
||||
|
||||
#include <primitives/transaction.h>
|
||||
#include <hash.h>
|
||||
|
||||
+20
-15
@@ -123,7 +123,7 @@ void CChainParams::UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64
|
||||
class CMainParams : public CChainParams {
|
||||
public:
|
||||
CMainParams() {
|
||||
strNetworkID = "lbrycrd";
|
||||
strNetworkID = CBaseChainParams::MAIN;
|
||||
consensus.nSubsidyLevelInterval = 1<<5;
|
||||
consensus.nMajorityEnforceBlockUpgrade = 750;
|
||||
consensus.nMajorityRejectBlockOutdated = 950;
|
||||
@@ -144,7 +144,9 @@ public:
|
||||
consensus.nAllowMinDiffMaxHeight = -1;
|
||||
consensus.nNormalizedNameForkHeight = 539940; // targeting 21 March 2019
|
||||
consensus.nMinTakeoverWorkaroundHeight = 496850;
|
||||
consensus.nMaxTakeoverWorkaroundHeight = 10000000;
|
||||
consensus.nMaxTakeoverWorkaroundHeight = 658300; // targeting 30 Oct 2019
|
||||
consensus.nWitnessForkHeight = 680770; // targeting 11 Dec 2019
|
||||
consensus.nAllClaimsInMerkleForkHeight = 658310; // targeting 30 Oct 2019
|
||||
consensus.fPowAllowMinDifficultyBlocks = false;
|
||||
consensus.fPowNoRetargeting = false;
|
||||
consensus.nRuleChangeActivationThreshold = 1916; // 95% of a half week
|
||||
@@ -158,17 +160,16 @@ public:
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 1462060800; // May 1st, 2016
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 1493596800; // May 1st, 2017
|
||||
|
||||
// Deployment of SegWit (BIP141, BIP143, and BIP147)
|
||||
// Deployment of SegWit (BIP141, BIP143, and BIP147) -- Unused (see nWitnessForkHeight).
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1;
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1547942400; // Jan 20, 2019
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1548288000; // Jan 24, 2019
|
||||
|
||||
// The best chain should have at least this much work.
|
||||
consensus.nMinimumChainWork = uint256S("0x000000000000000000000000000000000000000000000000607ca7e806c4c1e9"); //400000
|
||||
consensus.nMinimumChainWork = uint256S("000000000000000000000000000000000000000000000499ed6684d1bf6f6fd3"); //946000
|
||||
|
||||
// By default assume that the signatures in ancestors of this block are valid.
|
||||
//consensus.defaultAssumeValid = uint256S("0xf0e56e70782af63ccb49c76e852540688755869ba59ec68cac9c04a6b4d9f5ca"); //400000
|
||||
consensus.defaultAssumeValid = uint256S("0xa6bbb48f5343eb9b0287c22f3ea8b29f36cf10794a37f8a925a894d6f4519913"); //4000
|
||||
consensus.defaultAssumeValid = uint256S("0d3b537afe49820e1c6efc555463f955251b1293c6e5130137e1e25744431172"); //946000
|
||||
|
||||
/**
|
||||
* The message start string is designed to be unlikely to occur in normal data.
|
||||
@@ -194,9 +195,11 @@ public:
|
||||
vSeeds.clear();
|
||||
vFixedSeeds.clear();
|
||||
|
||||
vSeeds.emplace_back("dnsseed1.lbry.io"); // lbry.io
|
||||
vSeeds.emplace_back("dnsseed2.lbry.io"); // lbry.io
|
||||
vSeeds.emplace_back("dnsseed3.lbry.io"); // lbry.io
|
||||
vSeeds.emplace_back("dnsseed1.lbry.io"); // LBRY Inc
|
||||
vSeeds.emplace_back("dnsseed2.lbry.io"); // LBRY Inc
|
||||
vSeeds.emplace_back("dnsseed3.lbry.io"); // LBRY Inc
|
||||
vSeeds.emplace_back("seed.lbry.grin.io"); // Grin
|
||||
vSeeds.emplace_back("seed.allaboutlbc.com"); // Madiator2011
|
||||
|
||||
base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1, 0x55);
|
||||
base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1, 0x7a);
|
||||
@@ -204,8 +207,6 @@ public:
|
||||
base58Prefixes[EXT_PUBLIC_KEY] = {0x04, 0x88, 0xB2, 0x1E};
|
||||
base58Prefixes[EXT_SECRET_KEY] = {0x04, 0x88, 0xAD, 0xE4};
|
||||
|
||||
vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
|
||||
|
||||
bech32_hrp = "lbc";
|
||||
|
||||
vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
|
||||
@@ -240,7 +241,7 @@ public:
|
||||
class CTestNetParams : public CChainParams {
|
||||
public:
|
||||
CTestNetParams() {
|
||||
strNetworkID = "lbrycrdtest";
|
||||
strNetworkID = CBaseChainParams::TESTNET;
|
||||
consensus.nSubsidyLevelInterval = 1 << 5;
|
||||
consensus.nMajorityEnforceBlockUpgrade = 51;
|
||||
consensus.nMajorityRejectBlockOutdated = 75;
|
||||
@@ -261,7 +262,9 @@ public:
|
||||
consensus.nAllowMinDiffMaxHeight = 1100000;
|
||||
consensus.nNormalizedNameForkHeight = 993380; // targeting, 21 Feb 2019
|
||||
consensus.nMinTakeoverWorkaroundHeight = 99;
|
||||
consensus.nMaxTakeoverWorkaroundHeight = 10000000;
|
||||
consensus.nMaxTakeoverWorkaroundHeight = 1198550; // targeting 30 Sep 2019
|
||||
consensus.nWitnessForkHeight = 1198600;
|
||||
consensus.nAllClaimsInMerkleForkHeight = 1198560; // targeting 30 Sep 2019
|
||||
consensus.fPowAllowMinDifficultyBlocks = true;
|
||||
consensus.fPowNoRetargeting = false;
|
||||
consensus.nRuleChangeActivationThreshold = 1512; // 75% for testchains
|
||||
@@ -275,7 +278,7 @@ public:
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 1456790400; // March 1st, 2016
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 1493596800; // May 1st, 2017
|
||||
|
||||
// Deployment of SegWit (BIP141, BIP143, and BIP147)
|
||||
// Deployment of SegWit (BIP141, BIP143, and BIP147) -- Unused (see nWitnessForkHeight).
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1;
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1462060800; // May 1st 2016
|
||||
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1493596800; // May 1st 2017
|
||||
@@ -348,7 +351,7 @@ public:
|
||||
class CRegTestParams : public CChainParams {
|
||||
public:
|
||||
CRegTestParams() {
|
||||
strNetworkID = "lbrycrdreg";
|
||||
strNetworkID = CBaseChainParams::REGTEST;
|
||||
consensus.nSubsidyLevelInterval = 1 << 5;
|
||||
consensus.BIP16Exception = uint256();
|
||||
consensus.BIP34Height = 1000; // BIP34 is needed for validation_block_tests
|
||||
@@ -367,6 +370,8 @@ public:
|
||||
consensus.nNormalizedNameForkHeight = 250; // SDK depends upon this number
|
||||
consensus.nMinTakeoverWorkaroundHeight = -1;
|
||||
consensus.nMaxTakeoverWorkaroundHeight = -1;
|
||||
consensus.nWitnessForkHeight = 150;
|
||||
consensus.nAllClaimsInMerkleForkHeight = 350;
|
||||
consensus.fPowAllowMinDifficultyBlocks = false;
|
||||
consensus.fPowNoRetargeting = false;
|
||||
consensus.nRuleChangeActivationThreshold = 108; // 75% for testchains
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
const std::string CBaseChainParams::MAIN = "lbrycrd";
|
||||
const std::string CBaseChainParams::TESTNET = "lbrycrdtest";
|
||||
const std::string CBaseChainParams::REGTEST = "regtest";
|
||||
const std::string CBaseChainParams::REGTEST = "lbrycrdreg";
|
||||
|
||||
void SetupChainParamsBaseOptions()
|
||||
{
|
||||
|
||||
@@ -149,7 +149,7 @@ bool ProcessClaim(CClaimScriptOp& claimOp, CClaimTrieCache& trieCache, const CSc
|
||||
{
|
||||
int op;
|
||||
std::vector<std::vector<unsigned char> > vvchParams;
|
||||
if (!DecodeClaimScript(scriptPubKey, op, vvchParams))
|
||||
if (!DecodeClaimScript(scriptPubKey, op, vvchParams, trieCache.allowSupportMetadata()))
|
||||
return false;
|
||||
|
||||
switch (op) {
|
||||
|
||||
+12
-12
@@ -59,17 +59,17 @@ public:
|
||||
*/
|
||||
CClaimScriptAddOp(const COutPoint& point, CAmount nValue, int nHeight);
|
||||
/**
|
||||
* Implamention of OP_CLAIM_NAME handler
|
||||
* Implementation of OP_CLAIM_NAME handler
|
||||
* @see CClaimScriptOp::claimName
|
||||
*/
|
||||
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
|
||||
/**
|
||||
* Implamention of OP_UPDATE_CLAIM handler
|
||||
* Implementation of OP_UPDATE_CLAIM handler
|
||||
* @see CClaimScriptOp::updateClaim
|
||||
*/
|
||||
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
/**
|
||||
* Implamention of OP_SUPPORT_CLAIM handler
|
||||
* Implementation of OP_SUPPORT_CLAIM handler
|
||||
* @see CClaimScriptOp::supportClaim
|
||||
*/
|
||||
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
@@ -100,17 +100,17 @@ public:
|
||||
*/
|
||||
CClaimScriptUndoAddOp(const COutPoint& point, int nHeight);
|
||||
/**
|
||||
* Implamention of OP_CLAIM_NAME handler
|
||||
* Implementation of OP_CLAIM_NAME handler
|
||||
* @see CClaimScriptOp::claimName
|
||||
*/
|
||||
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
|
||||
/**
|
||||
* Implamention of OP_UPDATE_CLAIM handler
|
||||
* Implementation of OP_UPDATE_CLAIM handler
|
||||
* @see CClaimScriptOp::updateClaim
|
||||
*/
|
||||
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
/**
|
||||
* Implamention of OP_SUPPORT_CLAIM handler
|
||||
* Implementation of OP_SUPPORT_CLAIM handler
|
||||
* @see CClaimScriptOp::supportClaim
|
||||
*/
|
||||
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
@@ -141,17 +141,17 @@ public:
|
||||
*/
|
||||
CClaimScriptSpendOp(const COutPoint& point, int nHeight, int& nValidHeight);
|
||||
/**
|
||||
* Implamention of OP_CLAIM_NAME handler
|
||||
* Implementation of OP_CLAIM_NAME handler
|
||||
* @see CClaimScriptOp::claimName
|
||||
*/
|
||||
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
|
||||
/**
|
||||
* Implamention of OP_UPDATE_CLAIM handler
|
||||
* Implementation of OP_UPDATE_CLAIM handler
|
||||
* @see CClaimScriptOp::updateClaim
|
||||
*/
|
||||
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
/**
|
||||
* Implamention of OP_SUPPORT_CLAIM handler
|
||||
* Implementation of OP_SUPPORT_CLAIM handler
|
||||
* @see CClaimScriptOp::supportClaim
|
||||
*/
|
||||
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
@@ -184,17 +184,17 @@ public:
|
||||
*/
|
||||
CClaimScriptUndoSpendOp(const COutPoint& point, CAmount nValue, int nHeight, int nValidHeight);
|
||||
/**
|
||||
* Implamention of OP_CLAIM_NAME handler
|
||||
* Implementation of OP_CLAIM_NAME handler
|
||||
* @see CClaimScriptOp::claimName
|
||||
*/
|
||||
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
|
||||
/**
|
||||
* Implamention of OP_UPDATE_CLAIM handler
|
||||
* Implementation of OP_UPDATE_CLAIM handler
|
||||
* @see CClaimScriptOp::updateClaim
|
||||
*/
|
||||
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
/**
|
||||
* Implamention of OP_SUPPORT_CLAIM handler
|
||||
* Implementation of OP_SUPPORT_CLAIM handler
|
||||
* @see CClaimScriptOp::supportClaim
|
||||
*/
|
||||
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
|
||||
|
||||
+357
-363
File diff suppressed because it is too large.
Load diff
+185
-97
@@ -18,9 +18,8 @@
|
||||
#include <unordered_set>
|
||||
|
||||
// leveldb keys
|
||||
#define TRIE_NODE 'n' // deprecated
|
||||
#define TRIE_NODE_BY_HASH 'h'
|
||||
#define TRIE_NODE_BY_NAME 'g'
|
||||
#define TRIE_NODE 'n'
|
||||
#define TRIE_NODE_CHILDREN 'b'
|
||||
#define CLAIM_BY_ID 'i'
|
||||
#define CLAIM_QUEUE_ROW 'r'
|
||||
#define CLAIM_QUEUE_NAME_ROW 'm'
|
||||
@@ -36,10 +35,10 @@ struct CClaimValue
|
||||
{
|
||||
COutPoint outPoint;
|
||||
uint160 claimId;
|
||||
CAmount nAmount;
|
||||
CAmount nEffectiveAmount;
|
||||
int nHeight;
|
||||
int nValidAtHeight;
|
||||
CAmount nAmount = 0;
|
||||
CAmount nEffectiveAmount = 0;
|
||||
int nHeight = 0;
|
||||
int nValidAtHeight = 0;
|
||||
|
||||
CClaimValue() = default;
|
||||
|
||||
@@ -63,7 +62,6 @@ struct CClaimValue
|
||||
READWRITE(nAmount);
|
||||
READWRITE(nHeight);
|
||||
READWRITE(nValidAtHeight);
|
||||
READWRITE(nEffectiveAmount);
|
||||
}
|
||||
|
||||
bool operator<(const CClaimValue& other) const
|
||||
@@ -94,9 +92,9 @@ struct CSupportValue
|
||||
{
|
||||
COutPoint outPoint;
|
||||
uint160 supportedClaimId;
|
||||
CAmount nAmount;
|
||||
int nHeight;
|
||||
int nValidAtHeight;
|
||||
CAmount nAmount = 0;
|
||||
int nHeight = 0;
|
||||
int nValidAtHeight = 0;
|
||||
|
||||
CSupportValue() = default;
|
||||
|
||||
@@ -160,6 +158,17 @@ struct CClaimTrieData
|
||||
inline void SerializationOp(Stream& s, Operation ser_action)
|
||||
{
|
||||
READWRITE(hash);
|
||||
|
||||
if (ser_action.ForRead()) {
|
||||
if (s.eof()) {
|
||||
claims.clear();
|
||||
nHeightOfLastTakeover = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
else if (claims.empty())
|
||||
return;
|
||||
|
||||
READWRITE(claims);
|
||||
READWRITE(nHeightOfLastTakeover);
|
||||
}
|
||||
@@ -180,34 +189,10 @@ struct CClaimTrieData
|
||||
}
|
||||
};
|
||||
|
||||
struct CClaimTrieDataNode {
|
||||
CClaimTrieData data;
|
||||
// we're using a vector to avoid RAM thrashing and for faster serialization ops.
|
||||
// We're assuming its data is inserted in order and never modified.
|
||||
std::vector<std::pair<std::string, uint256>> children;
|
||||
bool childrenSerialization = true;
|
||||
|
||||
CClaimTrieDataNode() = default;
|
||||
CClaimTrieDataNode(CClaimTrieDataNode&&) = default;
|
||||
CClaimTrieDataNode(const CClaimTrieDataNode&) = default;
|
||||
CClaimTrieDataNode& operator=(CClaimTrieDataNode&&) = default;
|
||||
CClaimTrieDataNode& operator=(const CClaimTrieDataNode& d) = default;
|
||||
|
||||
ADD_SERIALIZE_METHODS;
|
||||
|
||||
template <typename Stream, typename Operation>
|
||||
inline void SerializationOp(Stream& s, Operation ser_action)
|
||||
{
|
||||
READWRITE(data);
|
||||
if (childrenSerialization) // wanting constexpr but hoping the compiler is smart enough anyway
|
||||
READWRITE(children);
|
||||
}
|
||||
};
|
||||
|
||||
struct COutPointHeightType
|
||||
{
|
||||
COutPoint outPoint;
|
||||
int nHeight;
|
||||
int nHeight = 0;
|
||||
|
||||
COutPointHeightType() = default;
|
||||
|
||||
@@ -230,7 +215,7 @@ struct CNameOutPointHeightType
|
||||
{
|
||||
std::string name;
|
||||
COutPoint outPoint;
|
||||
int nHeight;
|
||||
int nHeight = 0;
|
||||
|
||||
CNameOutPointHeightType() = default;
|
||||
|
||||
@@ -299,35 +284,73 @@ struct CClaimIndexElement
|
||||
CClaimValue claim;
|
||||
};
|
||||
|
||||
struct CClaimsForNameType
|
||||
struct CClaimNsupports
|
||||
{
|
||||
claimEntryType claims;
|
||||
supportEntryType supports;
|
||||
int nLastTakeoverHeight;
|
||||
std::string name;
|
||||
CClaimNsupports() = default;
|
||||
CClaimNsupports(CClaimNsupports&&) = default;
|
||||
CClaimNsupports(const CClaimNsupports&) = default;
|
||||
|
||||
CClaimsForNameType(claimEntryType claims, supportEntryType supports, int nLastTakeoverHeight, std::string name)
|
||||
: claims(std::move(claims)), supports(std::move(supports)), nLastTakeoverHeight(nLastTakeoverHeight), name(std::move(name))
|
||||
CClaimNsupports& operator=(CClaimNsupports&&) = default;
|
||||
CClaimNsupports& operator=(const CClaimNsupports&) = default;
|
||||
|
||||
CClaimNsupports(const CClaimValue& claim, CAmount effectiveAmount, const std::vector<CSupportValue>& supports = {})
|
||||
: claim(claim), effectiveAmount(effectiveAmount), supports(supports)
|
||||
{
|
||||
}
|
||||
|
||||
CClaimsForNameType(CClaimsForNameType&&) = default;
|
||||
CClaimsForNameType(const CClaimsForNameType&) = default;
|
||||
CClaimsForNameType& operator=(CClaimsForNameType&&) = default;
|
||||
CClaimsForNameType& operator=(const CClaimsForNameType&) = default;
|
||||
bool IsNull() const
|
||||
{
|
||||
return claim.claimId.IsNull();
|
||||
}
|
||||
|
||||
CClaimValue claim;
|
||||
CAmount effectiveAmount = 0;
|
||||
std::vector<CSupportValue> supports;
|
||||
};
|
||||
|
||||
class CClaimTrie
|
||||
{
|
||||
int nNextHeight = 0;
|
||||
int nProportionalDelayFactor = 0;
|
||||
std::unique_ptr<CDBWrapper> db;
|
||||
static const CClaimNsupports invalid;
|
||||
|
||||
struct CClaimSupportToName
|
||||
{
|
||||
CClaimSupportToName(const std::string& name, int nLastTakeoverHeight, std::vector<CClaimNsupports> claimsNsupports, std::vector<CSupportValue> unmatchedSupports)
|
||||
: name(name), nLastTakeoverHeight(nLastTakeoverHeight), claimsNsupports(std::move(claimsNsupports)), unmatchedSupports(std::move(unmatchedSupports))
|
||||
{
|
||||
}
|
||||
|
||||
const CClaimNsupports& find(const uint160& claimId) const
|
||||
{
|
||||
auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&claimId](const CClaimNsupports& value) {
|
||||
return claimId == value.claim.claimId;
|
||||
});
|
||||
return it != claimsNsupports.end() ? *it : invalid;
|
||||
}
|
||||
|
||||
const CClaimNsupports& find(const std::string& partialId) const
|
||||
{
|
||||
std::string lowered(partialId);
|
||||
for (auto& c: lowered)
|
||||
c = std::tolower(c);
|
||||
|
||||
auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&lowered](const CClaimNsupports& value) {
|
||||
return value.claim.claimId.GetHex().find(lowered) == 0;
|
||||
});
|
||||
return it != claimsNsupports.end() ? *it : invalid;
|
||||
}
|
||||
|
||||
const std::string name;
|
||||
const int nLastTakeoverHeight;
|
||||
const std::vector<CClaimNsupports> claimsNsupports;
|
||||
const std::vector<CSupportValue> unmatchedSupports;
|
||||
};
|
||||
|
||||
class CClaimTrie : public CPrefixTrie<std::string, CClaimTrieData>
|
||||
{
|
||||
public:
|
||||
CClaimTrie() = default;
|
||||
virtual ~CClaimTrie() = default;
|
||||
CClaimTrie(CClaimTrie&&) = delete;
|
||||
CClaimTrie(const CClaimTrie&) = delete;
|
||||
CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor = 32);
|
||||
CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor = 32, std::size_t cacheMB=200);
|
||||
|
||||
CClaimTrie& operator=(CClaimTrie&&) = delete;
|
||||
CClaimTrie& operator=(const CClaimTrie&) = delete;
|
||||
@@ -338,19 +361,17 @@ public:
|
||||
friend struct ClaimTrieChainFixture;
|
||||
friend class CClaimTrieCacheExpirationFork;
|
||||
friend class CClaimTrieCacheNormalizationFork;
|
||||
friend bool getClaimById(const uint160&, std::string&, CClaimValue*);
|
||||
friend bool getClaimById(const std::string&, std::string&, CClaimValue*);
|
||||
|
||||
std::size_t getTotalNamesInTrie() const;
|
||||
std::size_t getTotalClaimsInTrie() const;
|
||||
CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
|
||||
bool checkConsistency(const uint256& rootHash) const;
|
||||
|
||||
bool contains(const std::string& key) const;
|
||||
bool empty() const;
|
||||
bool find(const uint256& key, CClaimTrieDataNode& node) const;
|
||||
bool find(const std::string& key, CClaimTrieDataNode& node) const;
|
||||
|
||||
std::vector<std::pair<std::string, CClaimTrieDataNode>> nodes(const std::string& key) const;
|
||||
void recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<void(const std::string&, const CClaimTrieDataNode&)> function) const;
|
||||
protected:
|
||||
int nNextHeight = 0;
|
||||
int nProportionalDelayFactor = 0;
|
||||
std::unique_ptr<CDBWrapper> db;
|
||||
};
|
||||
|
||||
struct CClaimTrieProofNode
|
||||
@@ -373,21 +394,66 @@ struct CClaimTrieProofNode
|
||||
struct CClaimTrieProof
|
||||
{
|
||||
CClaimTrieProof() = default;
|
||||
|
||||
CClaimTrieProof(std::vector<CClaimTrieProofNode> nodes, bool hasValue, const COutPoint& outPoint, int nHeightOfLastTakeover)
|
||||
: nodes(std::move(nodes)), hasValue(hasValue), outPoint(outPoint), nHeightOfLastTakeover(nHeightOfLastTakeover)
|
||||
{
|
||||
}
|
||||
|
||||
CClaimTrieProof(CClaimTrieProof&&) = default;
|
||||
CClaimTrieProof(const CClaimTrieProof&) = default;
|
||||
CClaimTrieProof& operator=(CClaimTrieProof&&) = default;
|
||||
CClaimTrieProof& operator=(const CClaimTrieProof&) = default;
|
||||
|
||||
std::vector<std::pair<bool, uint256>> pairs;
|
||||
std::vector<CClaimTrieProofNode> nodes;
|
||||
bool hasValue;
|
||||
int nHeightOfLastTakeover = 0;
|
||||
bool hasValue = false;
|
||||
COutPoint outPoint;
|
||||
int nHeightOfLastTakeover;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class COptional
|
||||
{
|
||||
bool own;
|
||||
T* value;
|
||||
public:
|
||||
COptional(T* value = nullptr) : own(false), value(value) {}
|
||||
COptional(COptional&& o)
|
||||
{
|
||||
own = o.own;
|
||||
value = o.value;
|
||||
o.own = false;
|
||||
o.value = nullptr;
|
||||
}
|
||||
COptional(T&& o) : own(true)
|
||||
{
|
||||
value = new T(std::move(o));
|
||||
}
|
||||
~COptional()
|
||||
{
|
||||
if (own)
|
||||
delete value;
|
||||
}
|
||||
COptional& operator=(COptional&&) = delete;
|
||||
bool unique() const
|
||||
{
|
||||
return own;
|
||||
}
|
||||
operator bool() const
|
||||
{
|
||||
return value;
|
||||
}
|
||||
operator T*() const
|
||||
{
|
||||
return value;
|
||||
}
|
||||
T* operator->() const
|
||||
{
|
||||
return value;
|
||||
}
|
||||
operator T&() const
|
||||
{
|
||||
return *value;
|
||||
}
|
||||
T& operator*() const
|
||||
{
|
||||
return *value;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
@@ -410,8 +476,6 @@ typedef std::map<int, expirationQueueRowType> expirationQueueType;
|
||||
typedef std::set<CClaimValue> claimIndexClaimListType;
|
||||
typedef std::vector<CClaimIndexElement> claimIndexElementListType;
|
||||
|
||||
typedef CPrefixTrie<std::string, CClaimTrieData> CClaimPrefixTrie;
|
||||
|
||||
class CClaimTrieCacheBase
|
||||
{
|
||||
public:
|
||||
@@ -420,17 +484,16 @@ public:
|
||||
|
||||
uint256 getMerkleHash();
|
||||
|
||||
bool getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const;
|
||||
|
||||
bool flush();
|
||||
bool empty() const;
|
||||
bool checkConsistency() const;
|
||||
bool ReadFromDisk(const CBlockIndex* tip);
|
||||
|
||||
bool haveClaim(const std::string& name, const COutPoint& outPoint) const;
|
||||
bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
|
||||
bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
|
||||
|
||||
bool haveSupport(const std::string& name, const COutPoint& outPoint) const;
|
||||
bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
|
||||
bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
|
||||
|
||||
bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
|
||||
bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight);
|
||||
@@ -460,25 +523,24 @@ public:
|
||||
|
||||
virtual int expirationTime() const;
|
||||
|
||||
bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
|
||||
virtual bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
|
||||
|
||||
virtual CClaimsForNameType getClaimsForName(const std::string& name) const;
|
||||
virtual CClaimSupportToName getClaimsForName(const std::string& name) const;
|
||||
|
||||
CAmount getEffectiveAmountForClaim(const std::string& name, const uint160& claimId, std::vector<CSupportValue>* supports = nullptr) const;
|
||||
CAmount getEffectiveAmountForClaim(const CClaimsForNameType& claims, const uint160& claimId, std::vector<CSupportValue>* supports = nullptr) const;
|
||||
CClaimTrie::const_iterator find(const std::string& name) const;
|
||||
void iterate(std::function<void(const std::string&, const CClaimTrieData&)> callback) const;
|
||||
|
||||
CClaimPrefixTrie::const_iterator begin() const;
|
||||
CClaimPrefixTrie::const_iterator end() const;
|
||||
|
||||
void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const;
|
||||
void dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase = true) const;
|
||||
virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
|
||||
|
||||
protected:
|
||||
CClaimTrie* base;
|
||||
CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
|
||||
CClaimTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
|
||||
std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
|
||||
std::unordered_set<std::string> namesToCheckForTakeover; // takeover numbers are updated on increment
|
||||
|
||||
uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it);
|
||||
virtual uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it);
|
||||
virtual bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const;
|
||||
|
||||
virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover);
|
||||
virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover);
|
||||
@@ -491,7 +553,7 @@ protected:
|
||||
int getDelayForName(const std::string& name) const;
|
||||
virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
|
||||
|
||||
CClaimPrefixTrie::iterator cacheData(const std::string& name, bool create = true);
|
||||
CClaimTrie::iterator cacheData(const std::string& name, bool create = true);
|
||||
|
||||
bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const;
|
||||
|
||||
@@ -520,10 +582,8 @@ private:
|
||||
|
||||
std::unordered_map<std::string, supportEntryType> supportCache; // to be added/updated to base (and disk) on flush
|
||||
std::unordered_set<std::string> nodesToDelete; // to be removed from base (and disk) on flush
|
||||
std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
|
||||
std::unordered_map<std::string, bool> takeoverWorkaround;
|
||||
std::unordered_set<std::string> removalWorkaround;
|
||||
std::unordered_set<std::string> dirtyNodes;
|
||||
|
||||
bool shouldUseTakeoverWorkaround(const std::string& key) const;
|
||||
void addTakeoverWorkaroundPotential(const std::string& key);
|
||||
@@ -535,19 +595,26 @@ private:
|
||||
bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
|
||||
bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
|
||||
|
||||
bool validateTrieConsistency(const CBlockIndex* tip);
|
||||
template <typename T>
|
||||
void insertRowsFromQueue(std::vector<T>& result, const std::string& name) const;
|
||||
|
||||
template <typename T>
|
||||
std::pair<const int, std::vector<queueEntryType<T>>>* getQueueCacheRow(int nHeight, bool createIfNotExists = false);
|
||||
std::vector<queueEntryType<T>>* getQueueCacheRow(int nHeight, bool createIfNotExists);
|
||||
|
||||
template <typename T>
|
||||
typename queueNameType::value_type* getQueueCacheNameRow(const std::string& name, bool createIfNotExists = false);
|
||||
COptional<const std::vector<queueEntryType<T>>> getQueueCacheRow(int nHeight) const;
|
||||
|
||||
template <typename T>
|
||||
typename expirationQueueType::value_type* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists = false);
|
||||
queueNameRowType* getQueueCacheNameRow(const std::string& name, bool createIfNotExists);
|
||||
|
||||
template <typename T>
|
||||
bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
|
||||
COptional<const queueNameRowType> getQueueCacheNameRow(const std::string& name) const;
|
||||
|
||||
template <typename T>
|
||||
expirationQueueRowType* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists);
|
||||
|
||||
template <typename T>
|
||||
bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
|
||||
|
||||
template <typename T>
|
||||
T add(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
|
||||
@@ -595,7 +662,8 @@ public:
|
||||
void setExpirationTime(int time);
|
||||
int expirationTime() const override;
|
||||
|
||||
void expirationForkActive(int height, bool increment);
|
||||
virtual void initializeIncrement();
|
||||
bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
|
||||
|
||||
bool incrementBlock(insertUndoType& insertUndo,
|
||||
claimQueueRowType& expireUndo,
|
||||
@@ -640,7 +708,7 @@ public:
|
||||
|
||||
bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
|
||||
bool getInfoForName(const std::string& name, CClaimValue& claim) const override;
|
||||
CClaimsForNameType getClaimsForName(const std::string& name) const override;
|
||||
CClaimSupportToName getClaimsForName(const std::string& name) const override;
|
||||
std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override;
|
||||
|
||||
protected:
|
||||
@@ -663,6 +731,26 @@ private:
|
||||
std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
|
||||
};
|
||||
|
||||
typedef CClaimTrieCacheNormalizationFork CClaimTrieCache;
|
||||
class CClaimTrieCacheHashFork : public CClaimTrieCacheNormalizationFork
|
||||
{
|
||||
public:
|
||||
explicit CClaimTrieCacheHashFork(CClaimTrie* base);
|
||||
|
||||
bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
|
||||
bool getProofForName(const std::string& name, CClaimTrieProof& proof, const std::function<bool(const CClaimValue&)>& comp);
|
||||
void initializeIncrement() override;
|
||||
bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
|
||||
|
||||
bool allowSupportMetadata() const;
|
||||
|
||||
protected:
|
||||
uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it) override;
|
||||
bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const override;
|
||||
|
||||
private:
|
||||
void copyAllBaseToCache();
|
||||
};
|
||||
|
||||
typedef CClaimTrieCacheHashFork CClaimTrieCache;
|
||||
|
||||
#endif // BITCOIN_CLAIMTRIE_H
|
||||
+247
-15
@@ -1,9 +1,9 @@
|
||||
|
||||
#include <consensus/merkle.h>
|
||||
#include <chainparams.h>
|
||||
#include <claimtrie.h>
|
||||
#include <hash.h>
|
||||
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/locale.hpp>
|
||||
#include <boost/locale/conversion.hpp>
|
||||
#include <boost/locale/localization_backend.hpp>
|
||||
@@ -44,10 +44,21 @@ bool CClaimTrieCacheExpirationFork::decrementBlock(insertUndoType& insertUndo, c
|
||||
return false;
|
||||
}
|
||||
|
||||
void CClaimTrieCacheExpirationFork::expirationForkActive(int nHeight, bool increment)
|
||||
void CClaimTrieCacheExpirationFork::initializeIncrement()
|
||||
{
|
||||
if (nHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight)
|
||||
forkForExpirationChange(increment);
|
||||
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
|
||||
if (nNextHeight != Params().GetConsensus().nExtendedClaimExpirationForkHeight)
|
||||
return;
|
||||
|
||||
forkForExpirationChange(true);
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheExpirationFork::finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
|
||||
{
|
||||
auto ret = CClaimTrieCacheBase::finalizeDecrement(takeoverHeightUndo);
|
||||
if (ret && nNextHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight)
|
||||
forkForExpirationChange(false);
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
|
||||
@@ -161,17 +172,12 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert
|
||||
// run the one-time upgrade of all names that need to change
|
||||
// it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
|
||||
|
||||
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
|
||||
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
|
||||
std::pair<uint8_t, std::string> key;
|
||||
if (!pcursor->GetKey(key) || key.first != TRIE_NODE_BY_NAME)
|
||||
continue;
|
||||
|
||||
const auto& name = key.second;
|
||||
const std::string normalized = normalizeClaimName(name, true);
|
||||
if (normalized == key.second)
|
||||
for (auto it = base->cbegin(); it != base->cend(); ++it) {
|
||||
const std::string normalized = normalizeClaimName(it.key(), true);
|
||||
if (normalized == it.key())
|
||||
continue;
|
||||
|
||||
auto& name = it.key();
|
||||
auto supports = getSupportsForName(name);
|
||||
for (auto support : supports) {
|
||||
// if it's already going to expire just skip it
|
||||
@@ -233,7 +239,7 @@ bool CClaimTrieCacheNormalizationFork::getInfoForName(const std::string& name, C
|
||||
return CClaimTrieCacheExpirationFork::getInfoForName(normalizeClaimName(name), claim);
|
||||
}
|
||||
|
||||
CClaimsForNameType CClaimTrieCacheNormalizationFork::getClaimsForName(const std::string& name) const
|
||||
CClaimSupportToName CClaimTrieCacheNormalizationFork::getClaimsForName(const std::string& name) const
|
||||
{
|
||||
return CClaimTrieCacheExpirationFork::getClaimsForName(normalizeClaimName(name));
|
||||
}
|
||||
@@ -247,3 +253,229 @@ std::string CClaimTrieCacheNormalizationFork::adjustNameForValidHeight(const std
|
||||
{
|
||||
return normalizeClaimName(name, validHeight > Params().GetConsensus().nNormalizedNameForkHeight);
|
||||
}
|
||||
|
||||
CClaimTrieCacheHashFork::CClaimTrieCacheHashFork(CClaimTrie* base) : CClaimTrieCacheNormalizationFork(base)
|
||||
{
|
||||
}
|
||||
|
||||
static const uint256 leafHash = uint256S("0000000000000000000000000000000000000000000000000000000000000002");
|
||||
static const uint256 emptyHash = uint256S("0000000000000000000000000000000000000000000000000000000000000003");
|
||||
|
||||
std::vector<uint256> getClaimHashes(const CClaimTrieData& data)
|
||||
{
|
||||
std::vector<uint256> hashes;
|
||||
for (auto& claim : data.claims)
|
||||
hashes.push_back(getValueHash(claim.outPoint, data.nHeightOfLastTakeover));
|
||||
return hashes;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
using iCbType = std::function<uint256(T&)>;
|
||||
|
||||
template <typename TIterator>
|
||||
uint256 recursiveBinaryTreeHash(TIterator& it, const iCbType<TIterator>& process)
|
||||
{
|
||||
std::vector<uint256> childHashes;
|
||||
for (auto& child : it.children())
|
||||
childHashes.emplace_back(process(child));
|
||||
|
||||
std::vector<uint256> claimHashes;
|
||||
if (!it->empty())
|
||||
claimHashes = getClaimHashes(it.data());
|
||||
else if (!it.hasChildren())
|
||||
return {};
|
||||
|
||||
auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
|
||||
auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes);
|
||||
|
||||
return Hash(left.begin(), left.end(), right.begin(), right.end());
|
||||
}
|
||||
|
||||
uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(CClaimTrie::iterator& it)
|
||||
{
|
||||
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
|
||||
return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(it);
|
||||
|
||||
using iterator = CClaimTrie::iterator;
|
||||
iCbType<iterator> process = [&process](iterator& it) -> uint256 {
|
||||
if (it->hash.IsNull())
|
||||
it->hash = recursiveBinaryTreeHash(it, process);
|
||||
assert(!it->hash.IsNull());
|
||||
return it->hash;
|
||||
};
|
||||
return process(it);
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheHashFork::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const
|
||||
{
|
||||
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
|
||||
return CClaimTrieCacheNormalizationFork::recursiveCheckConsistency(it, failed);
|
||||
|
||||
struct CRecursiveBreak {};
|
||||
using iterator = CClaimTrie::const_iterator;
|
||||
iCbType<iterator> process = [&failed, &process](iterator& it) -> uint256 {
|
||||
if (it->hash.IsNull() || it->hash != recursiveBinaryTreeHash(it, process)) {
|
||||
failed = it.key();
|
||||
throw CRecursiveBreak();
|
||||
}
|
||||
return it->hash;
|
||||
};
|
||||
|
||||
try {
|
||||
process(it);
|
||||
} catch (const CRecursiveBreak&) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
|
||||
{
|
||||
uint32_t count = 0;
|
||||
int matchlevel = -1;
|
||||
bool matchh = false;
|
||||
uint256 inner[32], h;
|
||||
const uint32_t one = 1;
|
||||
std::vector<uint256> res;
|
||||
|
||||
const auto iterateInner = [&](int& level) {
|
||||
for (; !(count & (one << level)); level++) {
|
||||
const auto& ihash = inner[level];
|
||||
if (matchh) {
|
||||
res.push_back(ihash);
|
||||
} else if (matchlevel == level) {
|
||||
res.push_back(h);
|
||||
matchh = true;
|
||||
}
|
||||
h = Hash(ihash.begin(), ihash.end(), h.begin(), h.end());
|
||||
}
|
||||
};
|
||||
|
||||
while (count < hashes.size()) {
|
||||
h = hashes[count];
|
||||
matchh = count == idx;
|
||||
count++;
|
||||
int level = 0;
|
||||
iterateInner(level);
|
||||
// Store the resulting hash at inner position level.
|
||||
inner[level] = h;
|
||||
if (matchh)
|
||||
matchlevel = level;
|
||||
}
|
||||
|
||||
int level = 0;
|
||||
while (!(count & (one << level)))
|
||||
level++;
|
||||
|
||||
h = inner[level];
|
||||
matchh = matchlevel == level;
|
||||
|
||||
while (count != (one << level)) {
|
||||
// If we reach this point, h is an inner value that is not the top.
|
||||
if (matchh)
|
||||
res.push_back(h);
|
||||
|
||||
h = Hash(h.begin(), h.end(), h.begin(), h.end());
|
||||
// Increment count to the value it would have if two entries at this
|
||||
count += (one << level);
|
||||
level++;
|
||||
iterateInner(level);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof)
|
||||
{
|
||||
return getProofForName(name, proof, nullptr);
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof, const std::function<bool(const CClaimValue&)>& comp)
|
||||
{
|
||||
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
|
||||
return CClaimTrieCacheNormalizationFork::getProofForName(name, proof);
|
||||
|
||||
auto fillPairs = [&proof](const std::vector<uint256>& hashes, uint32_t idx) {
|
||||
auto partials = ComputeMerklePath(hashes, idx);
|
||||
for (int i = partials.size() - 1; i >= 0; --i)
|
||||
proof.pairs.emplace_back((idx >> i) & 1, partials[i]);
|
||||
};
|
||||
|
||||
// cache the parent nodes
|
||||
cacheData(name, false);
|
||||
getMerkleHash();
|
||||
proof = CClaimTrieProof();
|
||||
for (auto& it : static_cast<const CClaimTrie&>(nodesToAddOrUpdate).nodes(name)) {
|
||||
std::vector<uint256> childHashes;
|
||||
uint32_t nextCurrentIdx = 0;
|
||||
for (auto& child : it.children()) {
|
||||
if (name.find(child.key()) == 0)
|
||||
nextCurrentIdx = uint32_t(childHashes.size());
|
||||
childHashes.push_back(child->hash);
|
||||
}
|
||||
|
||||
std::vector<uint256> claimHashes;
|
||||
if (!it->empty())
|
||||
claimHashes = getClaimHashes(it.data());
|
||||
|
||||
// I am on a node; I need a hash(children, claims)
|
||||
// if I am the last node on the list, it will be hash(children, x)
|
||||
// else it will be hash(x, claims)
|
||||
if (it.key() == name) {
|
||||
uint32_t nClaimIndex = 0;
|
||||
auto& claims = it->claims;
|
||||
auto itClaim = !comp ? claims.begin() : std::find_if(claims.begin(), claims.end(), comp);
|
||||
if (itClaim != claims.end()) {
|
||||
proof.hasValue = true;
|
||||
proof.outPoint = itClaim->outPoint;
|
||||
proof.nHeightOfLastTakeover = it->nHeightOfLastTakeover;
|
||||
nClaimIndex = std::distance(claims.begin(), itClaim);
|
||||
}
|
||||
auto hash = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
|
||||
proof.pairs.emplace_back(true, hash);
|
||||
if (!claimHashes.empty())
|
||||
fillPairs(claimHashes, nClaimIndex);
|
||||
} else {
|
||||
auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes);
|
||||
proof.pairs.emplace_back(false, hash);
|
||||
if (!childHashes.empty())
|
||||
fillPairs(childHashes, nextCurrentIdx);
|
||||
}
|
||||
}
|
||||
std::reverse(proof.pairs.begin(), proof.pairs.end());
|
||||
return true;
|
||||
}
|
||||
|
||||
void CClaimTrieCacheHashFork::copyAllBaseToCache()
|
||||
{
|
||||
for (auto it = base->cbegin(); it != base->cend(); ++it)
|
||||
if (nodesAlreadyCached.insert(it.key()).second)
|
||||
nodesToAddOrUpdate.insert(it.key(), it.data());
|
||||
|
||||
for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it)
|
||||
it->hash.SetNull();
|
||||
}
|
||||
|
||||
void CClaimTrieCacheHashFork::initializeIncrement()
|
||||
{
|
||||
CClaimTrieCacheNormalizationFork::initializeIncrement();
|
||||
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
|
||||
if (nNextHeight != Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
|
||||
return;
|
||||
|
||||
// if we are forking, we load the entire base trie into the cache trie
|
||||
// we reset its hash computation so it can be recomputed completely
|
||||
copyAllBaseToCache();
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheHashFork::finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
|
||||
{
|
||||
auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement(takeoverHeightUndo);
|
||||
if (ret && nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
|
||||
copyAllBaseToCache();
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool CClaimTrieCacheHashFork::allowSupportMetadata() const
|
||||
{
|
||||
return nNextHeight >= Params().GetConsensus().nAllClaimsInMerkleForkHeight;
|
||||
}
|
||||
@@ -81,6 +81,8 @@ struct Params {
|
||||
int nMinTakeoverWorkaroundHeight;
|
||||
int nMaxTakeoverWorkaroundHeight;
|
||||
|
||||
int nWitnessForkHeight;
|
||||
|
||||
int64_t nPowTargetSpacing;
|
||||
int64_t nPowTargetTimespan;
|
||||
/** how long it took claims to expire before the hard fork */
|
||||
@@ -94,6 +96,8 @@ struct Params {
|
||||
nOriginalClaimExpirationTime :
|
||||
nExtendedClaimExpirationTime;
|
||||
}
|
||||
/** blocks before the hard fork that adds all claims into the merkle hash */
|
||||
int64_t nAllClaimsInMerkleForkHeight;
|
||||
int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; }
|
||||
uint256 nMinimumChainWork;
|
||||
uint256 defaultAssumeValid;
|
||||
|
||||
@@ -154,7 +154,8 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i
|
||||
const Coin& coin = inputs.AccessCoin(tx.vin[i].prevout);
|
||||
assert(!coin.IsSpent());
|
||||
const CTxOut &prevout = coin.out;
|
||||
nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags);
|
||||
const CScript& scriptPubKey = StripClaimScriptPrefix(prevout.scriptPubKey);
|
||||
nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, scriptPubKey, &tx.vin[i].scriptWitness, flags);
|
||||
}
|
||||
return nSigOps;
|
||||
}
|
||||
|
||||
+32
-14
@@ -15,6 +15,7 @@
|
||||
#include <util.h>
|
||||
#include <utilmoneystr.h>
|
||||
#include <utilstrencodings.h>
|
||||
#include <nameclaim.h>
|
||||
|
||||
UniValue ValueFromAmount(const CAmount& amount)
|
||||
{
|
||||
@@ -147,12 +148,20 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_address)
|
||||
out.pushKV("hex", HexStr(script.begin(), script.end()));
|
||||
|
||||
std::vector<std::vector<unsigned char>> solns;
|
||||
txnouttype type;
|
||||
Solver(script, type, solns);
|
||||
out.pushKV("type", GetTxnOutputType(type));
|
||||
txnouttype type; int claimOp;
|
||||
auto stripped = StripClaimScriptPrefix(script, claimOp);
|
||||
Solver(stripped, type, solns);
|
||||
if (claimOp >= 0) {
|
||||
out.pushKV("isclaim", UniValue(claimOp == OP_CLAIM_NAME || claimOp == OP_UPDATE_CLAIM));
|
||||
out.pushKV("issupport", UniValue(claimOp == OP_SUPPORT_CLAIM));
|
||||
out.pushKV("subtype", GetTxnOutputType(type));
|
||||
out.pushKV("type", GetTxnOutputType(TX_NONSTANDARD)); // trying to keep backwards compatibility
|
||||
}
|
||||
else
|
||||
out.pushKV("type", GetTxnOutputType(type)); // trying to keep backwards compatibility
|
||||
|
||||
CTxDestination address;
|
||||
if (include_address && ExtractDestination(script, address)) {
|
||||
if (include_address && ExtractDestination(stripped, address)) {
|
||||
out.pushKV("address", EncodeDestination(address));
|
||||
}
|
||||
}
|
||||
@@ -168,19 +177,28 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey,
|
||||
if (fIncludeHex)
|
||||
out.pushKV("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end()));
|
||||
|
||||
if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) {
|
||||
int claimOp;
|
||||
auto stripped = StripClaimScriptPrefix(scriptPubKey, claimOp);
|
||||
auto extracted = ExtractDestinations(stripped, type, addresses, nRequired);
|
||||
if (extracted)
|
||||
out.pushKV("reqSigs", nRequired);
|
||||
|
||||
if (claimOp >= 0) {
|
||||
out.pushKV("isclaim", UniValue(claimOp == OP_CLAIM_NAME || claimOp == OP_UPDATE_CLAIM));
|
||||
out.pushKV("issupport", UniValue(claimOp == OP_SUPPORT_CLAIM));
|
||||
out.pushKV("subtype", GetTxnOutputType(type));
|
||||
out.pushKV("type", GetTxnOutputType(TX_NONSTANDARD));
|
||||
}
|
||||
else
|
||||
out.pushKV("type", GetTxnOutputType(type));
|
||||
return;
|
||||
}
|
||||
|
||||
out.pushKV("reqSigs", nRequired);
|
||||
out.pushKV("type", GetTxnOutputType(type));
|
||||
|
||||
UniValue a(UniValue::VARR);
|
||||
for (const CTxDestination& addr : addresses) {
|
||||
a.push_back(EncodeDestination(addr));
|
||||
if (extracted) {
|
||||
UniValue a(UniValue::VARR);
|
||||
for (const CTxDestination &addr : addresses) {
|
||||
a.push_back(EncodeDestination(addr));
|
||||
}
|
||||
out.pushKV("addresses", a);
|
||||
}
|
||||
out.pushKV("addresses", a);
|
||||
}
|
||||
|
||||
void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags)
|
||||
|
||||
+14
-41
@@ -80,14 +80,14 @@ static void SetMaxOpenFiles(leveldb::Options *options) {
|
||||
// implementation that does not use extra file descriptors (the fds are
|
||||
// closed after being mmaped).
|
||||
//
|
||||
// Increasing the value beyond the default is dangerous because LevelDB will
|
||||
// fall back to a non-mmap implementation when the file count is too large.
|
||||
// Increasing the value beyond the nmap count is dangerous because LevelDB will
|
||||
// fall back to a non-mmap implementation when the file count is too large (thus contending select()).
|
||||
// On 32-bit Unix host we should decrease the value because the handles use
|
||||
// up real fds, and we want to avoid fd exhaustion issues.
|
||||
//
|
||||
// See PR #12495 for further discussion.
|
||||
|
||||
int default_open_files = options->max_open_files;
|
||||
int default_open_files = 400;
|
||||
#ifndef WIN32
|
||||
if (sizeof(void*) < 8) {
|
||||
options->max_open_files = 64;
|
||||
@@ -97,47 +97,13 @@ static void SetMaxOpenFiles(leveldb::Options *options) {
|
||||
options->max_open_files, default_open_files);
|
||||
}
|
||||
|
||||
class CappedLenCache: public leveldb::Cache {
|
||||
leveldb::Cache* inner;
|
||||
std::size_t maxKeyLen;
|
||||
public:
|
||||
CappedLenCache(std::size_t capacity, std::size_t maxKeyLen)
|
||||
: inner(leveldb::NewLRUCache(capacity)), maxKeyLen(maxKeyLen) {}
|
||||
|
||||
~CappedLenCache() override { delete inner; }
|
||||
|
||||
Handle* Insert(const leveldb::Slice& key, void* value, size_t charge,
|
||||
void (*deleter)(const leveldb::Slice& key, void* value)) override {
|
||||
if (key.size() <= maxKeyLen)
|
||||
return inner->Insert(key, value, charge, deleter);
|
||||
deleter(key, value);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Handle* Lookup(const leveldb::Slice& key) override { return inner->Lookup(key); }
|
||||
void Release(Handle* handle) override { return inner->Release(handle); }
|
||||
void* Value(Handle* handle) override { return inner->Value(handle); }
|
||||
void Erase(const leveldb::Slice& key) override {return inner->Erase(key); }
|
||||
uint64_t NewId() override { return inner->NewId(); }
|
||||
};
|
||||
|
||||
static leveldb::Options GetOptions(size_t nCacheSize)
|
||||
{
|
||||
leveldb::Options options;
|
||||
|
||||
options.filter_policy=leveldb::NewBloomFilterPolicy2(16);
|
||||
options.write_buffer_size=60 * 1024 * 1024;
|
||||
options.total_leveldb_mem=2500ULL * 1024ULL * 1024ULL;
|
||||
options.env=leveldb::Env::Default();
|
||||
options.compression = leveldb::kNoCompression;
|
||||
options.info_log = new CBitcoinLevelDBLogger();
|
||||
return options;
|
||||
|
||||
auto write_cache = std::min(nCacheSize / 4, size_t(4 * 1024 * 1024)); // cap write_cache at 4MB (default)
|
||||
auto write_cache = std::min(nCacheSize / 4, size_t(32 * 1024 * 1024)); // cap write_cache
|
||||
options.block_cache = leveldb::NewLRUCache(nCacheSize - write_cache * 2);
|
||||
// options.block_cache = new CappedLenCache(nCacheSize - write_cache * 2, 6);
|
||||
options.write_buffer_size = write_cache; // up to two write buffers may be held in memory simultaneously
|
||||
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
|
||||
options.filter_policy = leveldb::NewBloomFilterPolicy(12);
|
||||
options.compression = leveldb::kNoCompression;
|
||||
options.info_log = new CBitcoinLevelDBLogger();
|
||||
if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
|
||||
@@ -146,12 +112,11 @@ static leveldb::Options GetOptions(size_t nCacheSize)
|
||||
options.paranoid_checks = true;
|
||||
}
|
||||
SetMaxOpenFiles(&options);
|
||||
options.max_open_files = 30000;
|
||||
return options;
|
||||
}
|
||||
|
||||
CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
|
||||
: m_name(fs::basename(path))
|
||||
: m_name(fs::basename(path)), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION)
|
||||
{
|
||||
penv = nullptr;
|
||||
readoptions.verify_checksums = true;
|
||||
@@ -216,8 +181,16 @@ CDBWrapper::~CDBWrapper()
|
||||
options.env = nullptr;
|
||||
}
|
||||
|
||||
bool CDBWrapper::Sync() {
|
||||
CDBBatch batch(*this);
|
||||
return WriteBatch(batch, true);
|
||||
}
|
||||
|
||||
bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
|
||||
{
|
||||
if (!pdb)
|
||||
return false;
|
||||
|
||||
const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
|
||||
double mem_before = 0;
|
||||
if (log_memory) {
|
||||
|
||||
+102
-104
@@ -16,9 +16,6 @@
|
||||
#include <leveldb/db.h>
|
||||
#include <leveldb/write_batch.h>
|
||||
|
||||
static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
|
||||
static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
|
||||
|
||||
class dbwrapper_error : public std::runtime_error
|
||||
{
|
||||
public:
|
||||
@@ -33,87 +30,16 @@ namespace dbwrapper_private {
|
||||
|
||||
/** Handle database error by throwing dbwrapper_error exception.
|
||||
*/
|
||||
void HandleError(const leveldb::Status& status);
|
||||
void HandleError(const leveldb::Status& status);
|
||||
|
||||
/** Work around circular dependency, as well as for testing in dbwrapper_tests.
|
||||
* Database obfuscation should be considered an implementation detail of the
|
||||
* specific database.
|
||||
*/
|
||||
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
|
||||
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
|
||||
|
||||
};
|
||||
|
||||
/** Batch of changes queued to be written to a CDBWrapper */
|
||||
class CDBBatch
|
||||
{
|
||||
friend class CDBWrapper;
|
||||
|
||||
private:
|
||||
const CDBWrapper &parent;
|
||||
leveldb::WriteBatch batch;
|
||||
|
||||
CDataStream ssKey;
|
||||
CDataStream ssValue;
|
||||
|
||||
size_t size_estimate;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @param[in] _parent CDBWrapper that this batch is to be submitted to
|
||||
*/
|
||||
explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { };
|
||||
|
||||
void Clear()
|
||||
{
|
||||
batch.Clear();
|
||||
size_estimate = 0;
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void Write(const K& key, const V& value)
|
||||
{
|
||||
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
|
||||
ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE);
|
||||
ssValue << value;
|
||||
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
|
||||
leveldb::Slice slValue(ssValue.data(), ssValue.size());
|
||||
|
||||
batch.Put(slKey, slValue, nullptr);
|
||||
// LevelDB serializes writes as:
|
||||
// - byte: header
|
||||
// - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
|
||||
// - byte[]: key
|
||||
// - varint: value length
|
||||
// - byte[]: value
|
||||
// The formula below assumes the key and value are both less than 16k.
|
||||
size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
|
||||
ssKey.clear();
|
||||
ssValue.clear();
|
||||
}
|
||||
|
||||
template <typename K>
|
||||
void Erase(const K& key)
|
||||
{
|
||||
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
|
||||
batch.Delete(slKey);
|
||||
// LevelDB serializes erases as:
|
||||
// - byte: header
|
||||
// - varint: key length
|
||||
// - byte[]: key
|
||||
// The formula below assumes the key is less than 16kB.
|
||||
size_estimate += 2 + (slKey.size() > 127) + slKey.size();
|
||||
ssKey.clear();
|
||||
}
|
||||
|
||||
size_t SizeEstimate() const { return size_estimate; }
|
||||
};
|
||||
|
||||
class CDBIterator
|
||||
{
|
||||
private:
|
||||
@@ -127,7 +53,7 @@ public:
|
||||
* @param[in] _piter The original leveldb iterator.
|
||||
*/
|
||||
CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) :
|
||||
parent(_parent), piter(_piter) { };
|
||||
parent(_parent), piter(_piter) { };
|
||||
~CDBIterator();
|
||||
|
||||
bool Valid() const;
|
||||
@@ -136,7 +62,6 @@ public:
|
||||
|
||||
template<typename K> void Seek(const K& key) {
|
||||
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
|
||||
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
piter->Seek(slKey);
|
||||
@@ -173,6 +98,8 @@ public:
|
||||
|
||||
};
|
||||
|
||||
class CDBBatch;
|
||||
|
||||
class CDBWrapper
|
||||
{
|
||||
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
|
||||
@@ -213,6 +140,8 @@ private:
|
||||
std::vector<unsigned char> CreateObfuscateKey() const;
|
||||
|
||||
public:
|
||||
mutable CDataStream ssKey, ssValue;
|
||||
|
||||
/**
|
||||
* @param[in] path Location in the filesystem where leveldb data will be stored.
|
||||
* @param[in] nCacheSize Configures various leveldb cache settings.
|
||||
@@ -222,7 +151,7 @@ public:
|
||||
* with a zero'd byte array.
|
||||
*/
|
||||
CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
|
||||
~CDBWrapper();
|
||||
virtual ~CDBWrapper();
|
||||
|
||||
CDBWrapper(const CDBWrapper&) = delete;
|
||||
/* CDBWrapper& operator=(const CDBWrapper&) = delete; */
|
||||
@@ -230,13 +159,13 @@ public:
|
||||
template <typename K, typename V>
|
||||
bool Read(const K& key, V& value) const
|
||||
{
|
||||
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
|
||||
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
assert(ssKey.empty());
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
|
||||
std::string strValue;
|
||||
leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
|
||||
ssKey.clear();
|
||||
if (!status.ok()) {
|
||||
if (status.IsNotFound())
|
||||
return false;
|
||||
@@ -254,23 +183,17 @@ public:
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
bool Write(const K& key, const V& value, bool fSync = false)
|
||||
{
|
||||
CDBBatch batch(*this);
|
||||
batch.Write(key, value);
|
||||
return WriteBatch(batch, fSync);
|
||||
}
|
||||
bool Write(const K& key, const V& value, bool fSync = false);
|
||||
|
||||
template <typename K>
|
||||
bool Exists(const K& key) const
|
||||
{
|
||||
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
|
||||
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
|
||||
std::string strValue;
|
||||
leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
|
||||
ssKey.clear();
|
||||
if (!status.ok()) {
|
||||
if (status.IsNotFound())
|
||||
return false;
|
||||
@@ -281,12 +204,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename K>
|
||||
bool Erase(const K& key, bool fSync = false)
|
||||
{
|
||||
CDBBatch batch(*this);
|
||||
batch.Erase(key);
|
||||
return WriteBatch(batch, fSync);
|
||||
}
|
||||
bool Erase(const K& key, bool fSync = false);
|
||||
|
||||
bool WriteBatch(CDBBatch& batch, bool fSync = false);
|
||||
|
||||
@@ -299,11 +217,7 @@ public:
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Sync()
|
||||
{
|
||||
CDBBatch batch(*this);
|
||||
return WriteBatch(batch, true);
|
||||
}
|
||||
bool Sync();
|
||||
|
||||
CDBIterator *NewIterator()
|
||||
{
|
||||
@@ -319,8 +233,8 @@ public:
|
||||
size_t EstimateSize(const K& key_begin, const K& key_end) const
|
||||
{
|
||||
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
|
||||
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey1.reserve(ssKey.capacity());
|
||||
ssKey2.reserve(ssKey.capacity());
|
||||
ssKey1 << key_begin;
|
||||
ssKey2 << key_end;
|
||||
leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
|
||||
@@ -338,8 +252,8 @@ public:
|
||||
void CompactRange(const K& key_begin, const K& key_end) const
|
||||
{
|
||||
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
|
||||
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
|
||||
ssKey1.reserve(ssKey.capacity());
|
||||
ssKey2.reserve(ssKey.capacity());
|
||||
ssKey1 << key_begin;
|
||||
ssKey2 << key_end;
|
||||
leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
|
||||
@@ -349,4 +263,88 @@ public:
|
||||
|
||||
};
|
||||
|
||||
/** Batch of changes queued to be written to a CDBWrapper */
|
||||
class CDBBatch
|
||||
{
|
||||
friend class CDBWrapper;
|
||||
const CDBWrapper &parent;
|
||||
leveldb::WriteBatch batch;
|
||||
|
||||
size_t size_estimate;
|
||||
CDataStream ssKey, ssValue;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @param[in] _parent CDBWrapper that this batch is to be submitted to
|
||||
*/
|
||||
explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), size_estimate(0),
|
||||
ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) {
|
||||
ssKey.reserve(parent.ssKey.capacity());
|
||||
ssValue.reserve(parent.ssValue.capacity());
|
||||
};
|
||||
|
||||
void Clear()
|
||||
{
|
||||
batch.Clear();
|
||||
size_estimate = 0;
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void Write(const K& key, const V& value)
|
||||
{
|
||||
assert(ssKey.empty());
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
|
||||
assert(ssValue.empty());
|
||||
ssValue << value;
|
||||
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
|
||||
leveldb::Slice slValue(ssValue.data(), ssValue.size());
|
||||
|
||||
batch.Put(slKey, slValue);
|
||||
// LevelDB serializes writes as:
|
||||
// - byte: header
|
||||
// - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
|
||||
// - byte[]: key
|
||||
// - varint: value length
|
||||
// - byte[]: value
|
||||
// The formula below assumes the key and value are both less than 16k.
|
||||
size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
|
||||
ssKey.clear();
|
||||
ssValue.clear();
|
||||
}
|
||||
|
||||
template <typename K>
|
||||
void Erase(const K& key)
|
||||
{
|
||||
ssKey << key;
|
||||
leveldb::Slice slKey(ssKey.data(), ssKey.size());
|
||||
|
||||
batch.Delete(slKey);
|
||||
// LevelDB serializes erases as:
|
||||
// - byte: header
|
||||
// - varint: key length
|
||||
// - byte[]: key
|
||||
// The formula below assumes the key is less than 16kB.
|
||||
size_estimate += 2 + (slKey.size() > 127) + slKey.size();
|
||||
ssKey.clear();
|
||||
}
|
||||
|
||||
size_t SizeEstimate() const { return size_estimate; }
|
||||
};
|
||||
|
||||
template<typename K>
|
||||
bool CDBWrapper::Erase(const K &key, bool fSync) {
|
||||
CDBBatch batch(*this);
|
||||
batch.Erase(key);
|
||||
return WriteBatch(batch, fSync);
|
||||
}
|
||||
|
||||
template<typename K, typename V>
|
||||
bool CDBWrapper::Write(const K &key, const V &value, bool fSync) {
|
||||
CDBBatch batch(*this);
|
||||
batch.Write(key, value);
|
||||
return WriteBatch(batch, fSync);
|
||||
}
|
||||
|
||||
#endif // BITCOIN_DBWRAPPER_H
|
||||
@@ -28,6 +28,8 @@ protected:
|
||||
DB(const fs::path& path, size_t n_cache_size,
|
||||
bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
|
||||
|
||||
~DB() override {}
|
||||
|
||||
/// Read block locator of the chain that the txindex is in sync with.
|
||||
bool ReadBestBlock(CBlockLocator& locator) const;
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ class TxIndex::DB : public BaseIndex::DB
|
||||
{
|
||||
public:
|
||||
explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
|
||||
~DB() override {}
|
||||
|
||||
/// Read the disk location of the transaction data with the given hash. Returns false if the
|
||||
/// transaction hash is not indexed.
|
||||
|
||||
+9
-7
@@ -22,6 +22,7 @@
|
||||
#include <httprpc.h>
|
||||
#include <index/txindex.h>
|
||||
#include <key.h>
|
||||
#include <lbry.h>
|
||||
#include <validation.h>
|
||||
#include <miner.h>
|
||||
#include <netbase.h>
|
||||
@@ -370,6 +371,7 @@ void SetupServerArgs()
|
||||
gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-claimtriecache=<n>", strprintf("Set claim trie cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS);
|
||||
@@ -397,6 +399,7 @@ void SetupServerArgs()
|
||||
hidden_args.emplace_back("-sysperms");
|
||||
#endif
|
||||
gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
|
||||
gArgs.AddArg("-memfile=<GiB>", "Use a memory mapped file for the claimtrie allocations (default: use RAM instead)", false, OptionsCategory::OPTIONS);
|
||||
|
||||
gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
|
||||
gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
|
||||
@@ -483,7 +486,6 @@ void SetupServerArgs()
|
||||
CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST);
|
||||
gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), true, OptionsCategory::DEBUG_TEST);
|
||||
gArgs.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", false, OptionsCategory::DEBUG_TEST);
|
||||
gArgs.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", false, OptionsCategory::DEBUG_TEST);
|
||||
gArgs.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", false, OptionsCategory::DEBUG_TEST);
|
||||
|
||||
SetupChainParamsBaseOptions();
|
||||
@@ -1233,11 +1235,6 @@ bool AppInitMain()
|
||||
CreatePidFile(GetPidFile(), getpid());
|
||||
#endif
|
||||
if (g_logger->m_print_to_file) {
|
||||
if (gArgs.GetBoolArg("-shrinkdebugfile", g_logger->DefaultShrinkDebugFile())) {
|
||||
// Do this first since it both loads a bunch of debug.log into memory,
|
||||
// and because this needs to happen before any other debug.log printing
|
||||
g_logger->ShrinkDebugFile();
|
||||
}
|
||||
if (!g_logger->OpenDebugLog()) {
|
||||
return InitError(strprintf("Could not open debug log file %s",
|
||||
g_logger->m_file_path.string()));
|
||||
@@ -1440,6 +1437,8 @@ bool AppInitMain()
|
||||
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
|
||||
LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
|
||||
|
||||
g_memfileSize = gArgs.GetArg("-memfile", 0u);
|
||||
|
||||
bool fLoaded = false;
|
||||
while (!fLoaded && !ShutdownRequested()) {
|
||||
bool fReset = fReindex;
|
||||
@@ -1461,7 +1460,10 @@ bool AppInitMain()
|
||||
pblocktree.reset();
|
||||
pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset));
|
||||
delete pclaimTrie;
|
||||
pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState);
|
||||
int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache);
|
||||
trieCacheMB = std::min(trieCacheMB, nMaxDbCache);
|
||||
trieCacheMB = std::max(trieCacheMB, nMinDbCache);
|
||||
pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState, 32, trieCacheMB);
|
||||
|
||||
if (fReset) {
|
||||
pblocktree->WriteReindexing(true);
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
uint32_t g_memfileSize = 0;
|
||||
|
||||
unsigned int CalculateLbryNextWorkRequired(const CBlockIndex* pindexLast, int64_t nFirstBlockTime, const Consensus::Params& params)
|
||||
{
|
||||
if (params.fPowNoRetargeting)
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <chain.h>
|
||||
#include <chainparams.h>
|
||||
|
||||
extern uint32_t g_memfileSize;
|
||||
unsigned int CalculateLbryNextWorkRequired(const CBlockIndex* pindexLast, int64_t nLastRetargetTime, const Consensus::Params& params);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,13 @@
|
||||
build_config.mk
|
||||
*.a
|
||||
*.o
|
||||
*.dylib*
|
||||
*.so
|
||||
*.so.*
|
||||
*_test
|
||||
db_bench
|
||||
leveldbutil
|
||||
Release
|
||||
Debug
|
||||
Benchmark
|
||||
vs2010.*
|
||||
@@ -6,3 +6,7 @@ Google Inc.
|
||||
# Initial version authors:
|
||||
Jeffrey Dean <jeff@google.com>
|
||||
Sanjay Ghemawat <sanjay@google.com>
|
||||
|
||||
# Partial list of contributors:
|
||||
Kevin Regan <kevin.d.regan@gmail.com>
|
||||
Johan Bilien <jobi@litl.com>
|
||||
@@ -1,72 +0,0 @@
|
||||
github.com tag 2.0.34 - February 15, 2017
|
||||
-----------------------------------------
|
||||
mv-hot-backup2: - correct MakeTieredDbname() within db/filename.cc
|
||||
for case where dbname input is blank and fast/slow
|
||||
already populated in options. Corrects issue
|
||||
with hot backup in non-tiered storage situations
|
||||
|
||||
github.com tag 2.0.33 - November 21, 2016
|
||||
-----------------------------------------
|
||||
mv-bucket-expiry: - partial branch to enable X-Riak-Meta-Expiry-Base-Seconds
|
||||
property within enterprise edition
|
||||
|
||||
--- no 2.0.32 tag on leveldb ---
|
||||
|
||||
github.com tag 2.0.31 - November 1, 2016
|
||||
----------------------------------------
|
||||
- version shipped with Riak 2.2
|
||||
mv-no-md-expiry: - Riak specific
|
||||
- never convert a key prefix of sext:encoded "{md" to expiry
|
||||
- update sst_scan for dumping Riak formated keys
|
||||
mv-tuning8: - rework penalty rules in version_set.cc UpdatePenalty()
|
||||
- add unit test framework for UpdatePenalty()
|
||||
|
||||
github.com tag 2.0.30 - October 11, 2016
|
||||
----------------------------------------
|
||||
mv-delayed-bloom: - when opening an .sst table file, only load
|
||||
bloom filter on second Get() operation. Saves time.
|
||||
- correct VersionSet::Finalize() logic for level 1 when
|
||||
when level 2 is above desired size
|
||||
- move hot backup to Riak ee build
|
||||
|
||||
github.com tag 2.0.29 - September 13, 2016
|
||||
------------------------------------------
|
||||
mv-expiry-manifest: only switch to expiry enabled manifest format
|
||||
if expiry function enabled. Eases downgrade
|
||||
during early Riak releases containing expiry
|
||||
|
||||
github.com tag 2.0.28 - September 6, 2016
|
||||
-----------------------------------------
|
||||
mv-hot-backup: add externally triggered hot backup feature
|
||||
|
||||
github.com tag 2.0.27 - August 22, 2016
|
||||
---------------------------------------
|
||||
mv-mem-fences: fix iterator double delete bug in eleveldb and
|
||||
build better memory fenced operations for referenced count objects.
|
||||
|
||||
github.com tag 2.0.26 - August 21, 2016
|
||||
---------------------------------------
|
||||
mv-expiry-iter-bug: DBImpl::NewIterator() was not setting the new expiry parameter.
|
||||
|
||||
github.com tag 2.0.25 - August 10, 2016
|
||||
---------------------------------------
|
||||
Make LZ4 the default compression instead of Snappy.
|
||||
|
||||
github.com tag 2.0.24 - August 2, 2016
|
||||
--------------------------------------
|
||||
mv-expiry: open source expiry. Supports one expiry policy for all databases.
|
||||
|
||||
github.com tag 2.0.23 - July 20, 2016
|
||||
-------------------------------------
|
||||
mv-no-semaphore: remove semaphore controlled thread in hot_threads.cc. Instead use
|
||||
use mutex of thread 0 (only one thread's mutex) to address know race condition.
|
||||
|
||||
github.com tag 2.0.22 - June 22, 2016
|
||||
-------------------------------------
|
||||
no change: iterator fix in eleveldb
|
||||
|
||||
github.com tag 2.0.21 - June 16, 2016
|
||||
-------------------------------------
|
||||
branch mv-iterator-hot-threads: correct condition where eleveldb MoveTask
|
||||
could hang an iterator. (https://github.com/basho/leveldb/wiki/mv-iterator-hot-threads)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Contributing
|
||||
|
||||
We'd love to accept your code patches! However, before we can take them, we
|
||||
have to jump a couple of legal hurdles.
|
||||
|
||||
## Contributor License Agreements
|
||||
|
||||
Please fill out either the individual or corporate Contributor License
|
||||
Agreement as appropriate.
|
||||
|
||||
* If you are an individual writing original source code and you're sure you
|
||||
own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual).
|
||||
* If you work for a company that wants to allow you to contribute your work,
|
||||
then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate).
|
||||
|
||||
Follow either of the two links above to access the appropriate CLA and
|
||||
instructions for how to sign and return it.
|
||||
|
||||
## Submitting a Patch
|
||||
|
||||
1. Sign the contributors license agreement above.
|
||||
2. Decide which code you want to submit. A submission should be a set of changes
|
||||
that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues).
|
||||
Please don't mix more than one logical change per submission, because it makes
|
||||
the history hard to follow. If you want to make a change
|
||||
(e.g. add a sample or feature) that doesn't have a corresponding issue in the
|
||||
issue tracker, please create one.
|
||||
3. **Submitting**: When you are ready to submit, send us a Pull Request. Be
|
||||
sure to include the issue number you fixed and the name you used to sign
|
||||
the CLA.
|
||||
|
||||
## Writing Code ##
|
||||
|
||||
If your contribution contains code, please make sure that it follows
|
||||
[the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml).
|
||||
Otherwise we will have to ask you to make changes, and that's no fun for anyone.
|
||||
+362
-158
@@ -2,219 +2,423 @@
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
# Inherit some settings from environment variables, if available
|
||||
INSTALL_PATH ?= $(CURDIR)
|
||||
|
||||
#-----------------------------------------------
|
||||
# Uncomment exactly one of the lines labelled (A), (B), and (C) below
|
||||
# to switch between compilation modes.
|
||||
# NOTE: targets "debug" and "prof" provide same functionality
|
||||
# NOTE 2: -DNDEBUG disables assert() statements within C code,
|
||||
# i.e. no assert()s in production code
|
||||
|
||||
OPT ?= -O2 -g -DNDEBUG # (A) Production use (optimized mode)
|
||||
# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols
|
||||
# OPT ?= -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols
|
||||
# (A) Production use (optimized mode)
|
||||
OPT ?= -O2 -DNDEBUG
|
||||
# (B) Debug mode, w/ full line-level debugging symbols
|
||||
# OPT ?= -g2
|
||||
# (C) Profiling mode: opt, but w/debugging symbols
|
||||
# OPT ?= -O2 -g2 -DNDEBUG
|
||||
#-----------------------------------------------
|
||||
|
||||
# detect what platform we're building on
|
||||
ifeq ($(wildcard build_config.mk),)
|
||||
$(shell ./build_detect_platform build_config.mk)
|
||||
endif
|
||||
$(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \
|
||||
./build_detect_platform build_config.mk ./)
|
||||
# this file is generated by the previous line to set build flags and sources
|
||||
include build_config.mk
|
||||
|
||||
TESTS = \
|
||||
db/autocompact_test \
|
||||
db/c_test \
|
||||
db/corruption_test \
|
||||
db/db_test \
|
||||
db/dbformat_test \
|
||||
db/fault_injection_test \
|
||||
db/filename_test \
|
||||
db/log_test \
|
||||
db/recovery_test \
|
||||
db/skiplist_test \
|
||||
db/version_edit_test \
|
||||
db/version_set_test \
|
||||
db/write_batch_test \
|
||||
helpers/memenv/memenv_test \
|
||||
issues/issue178_test \
|
||||
issues/issue200_test \
|
||||
table/filter_block_test \
|
||||
table/table_test \
|
||||
util/arena_test \
|
||||
util/bloom_test \
|
||||
util/cache_test \
|
||||
util/coding_test \
|
||||
util/crc32c_test \
|
||||
util/env_posix_test \
|
||||
util/env_test \
|
||||
util/hash_test
|
||||
|
||||
UTILS = \
|
||||
db/db_bench \
|
||||
db/leveldbutil
|
||||
|
||||
# Put the object files in a subdirectory, but the application at the top of the object dir.
|
||||
PROGNAMES := $(notdir $(TESTS) $(UTILS))
|
||||
|
||||
# On Linux may need libkyotocabinet-dev for dependency.
|
||||
BENCHMARKS = \
|
||||
doc/bench/db_bench_sqlite3 \
|
||||
doc/bench/db_bench_tree_db
|
||||
|
||||
CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
|
||||
CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)
|
||||
|
||||
LDFLAGS += $(PLATFORM_LDFLAGS)
|
||||
LIBS += $(PLATFORM_LIBS)
|
||||
|
||||
LIBOBJECTS := $(SOURCES:.cc=.o)
|
||||
LIBOBJECTS += util/lz4.o
|
||||
MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
|
||||
DEPEND := $(SOURCES:.cc=.d)
|
||||
SIMULATOR_OUTDIR=out-ios-x86
|
||||
DEVICE_OUTDIR=out-ios-arm
|
||||
|
||||
TESTUTIL = ./util/testutil.o
|
||||
TESTHARNESS = ./util/testharness.o $(TESTUTIL)
|
||||
|
||||
TESTS := $(sort $(notdir $(basename $(TEST_SOURCES))))
|
||||
|
||||
TOOLS = \
|
||||
leveldb_repair \
|
||||
perf_dump \
|
||||
sst_rewrite \
|
||||
sst_scan
|
||||
|
||||
PROGRAMS = db_bench $(TESTS) $(TOOLS)
|
||||
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
|
||||
|
||||
LIBRARY = libleveldb.a
|
||||
MEMENVLIBRARY = libmemenv.a
|
||||
|
||||
#
|
||||
# static link leveldb to tools to simplify platform usage (if Linux)
|
||||
#
|
||||
ifeq ($(PLATFORM),OS_LINUX)
|
||||
LEVEL_LDFLAGS := -L . -Wl,-non_shared -lleveldb -Wl,-call_shared
|
||||
ifeq ($(PLATFORM), IOS)
|
||||
# Note: iOS should probably be using libtool, not ar.
|
||||
AR=xcrun ar
|
||||
SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path)
|
||||
DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path)
|
||||
DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64
|
||||
SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64
|
||||
STATIC_OUTDIR=out-ios-universal
|
||||
else
|
||||
LEVEL_LDFLAGS := -L . -lleveldb
|
||||
STATIC_OUTDIR=out-static
|
||||
SHARED_OUTDIR=out-shared
|
||||
STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES))
|
||||
SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench)
|
||||
endif
|
||||
|
||||
STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o))
|
||||
STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
|
||||
|
||||
DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o))
|
||||
DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
|
||||
|
||||
SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o))
|
||||
SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
|
||||
|
||||
SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o))
|
||||
SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
|
||||
|
||||
TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o
|
||||
TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL)
|
||||
|
||||
STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS)))
|
||||
STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS)))
|
||||
STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS)
|
||||
DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS)
|
||||
SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS)
|
||||
|
||||
default: all
|
||||
|
||||
# Should we build shared libraries?
|
||||
ifneq ($(PLATFORM_SHARED_EXT),)
|
||||
|
||||
# Many leveldb test apps use non-exported API's. Only build a subset for testing.
|
||||
SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS)
|
||||
|
||||
ifneq ($(PLATFORM_SHARED_VERSIONED),true)
|
||||
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED2 = $(SHARED1)
|
||||
SHARED3 = $(SHARED1)
|
||||
SHARED = $(SHARED1)
|
||||
SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED_LIB2 = $(SHARED_LIB1)
|
||||
SHARED_LIB3 = $(SHARED_LIB1)
|
||||
SHARED_LIBS = $(SHARED_LIB1)
|
||||
SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
|
||||
else
|
||||
# Update db.h if you change these.
|
||||
SHARED_MAJOR = 1
|
||||
SHARED_MINOR = 9
|
||||
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
|
||||
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
|
||||
SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
|
||||
$(SHARED1): $(SHARED3)
|
||||
ln -fs $(SHARED3) $(SHARED1)
|
||||
$(SHARED2): $(SHARED3)
|
||||
ln -fs $(SHARED3) $(SHARED2)
|
||||
SHARED_VERSION_MAJOR = 1
|
||||
SHARED_VERSION_MINOR = 20
|
||||
SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR)
|
||||
SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR)
|
||||
SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3)
|
||||
$(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3)
|
||||
ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1)
|
||||
$(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3)
|
||||
ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2)
|
||||
SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
|
||||
endif
|
||||
|
||||
$(SHARED3): $(LIBOBJECTS)
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIBOBJECTS) -o $(SHARED3) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2)
|
||||
$(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS)
|
||||
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS)
|
||||
|
||||
endif # PLATFORM_SHARED_EXT
|
||||
|
||||
all: $(SHARED) $(LIBRARY)
|
||||
|
||||
test check: all $(PROGRAMS) $(TESTS)
|
||||
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
|
||||
|
||||
tools: all $(TOOLS)
|
||||
|
||||
#
|
||||
# command line targets: debug and prof
|
||||
# just like
|
||||
ifneq ($(filter debug,$(MAKECMDGOALS)),)
|
||||
OPT := -g2 # (B) Debug mode, w/ full line-level debugging symbols
|
||||
debug: all
|
||||
endif
|
||||
|
||||
ifneq ($(filter prof,$(MAKECMDGOALS)),)
|
||||
OPT := -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols
|
||||
prof: all
|
||||
endif
|
||||
all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS)
|
||||
|
||||
check: $(STATIC_PROGRAMS)
|
||||
for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done
|
||||
|
||||
clean:
|
||||
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o */*.d */*/*.d ios-x86/*/*.o ios-arm/*/*.o build_config.mk include/leveldb/ldb_config.h
|
||||
-rm -rf ios-x86/* ios-arm/* *.dSYM
|
||||
-rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal
|
||||
-rm -f build_config.mk
|
||||
-rm -rf ios-x86 ios-arm
|
||||
|
||||
$(STATIC_OUTDIR):
|
||||
mkdir $@
|
||||
|
||||
$(LIBRARY): $(LIBOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(LIBOBJECTS)
|
||||
$(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
#
|
||||
# all tools, programs, and tests depend upon the static library
|
||||
$(TESTS) $(PROGRAMS) $(TOOLS) : $(LIBRARY)
|
||||
$(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR)
|
||||
mkdir -p $@
|
||||
|
||||
#
|
||||
# all tests depend upon the test harness
|
||||
$(TESTS) : $(TESTHARNESS)
|
||||
$(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
#
|
||||
# tools, programs, and tests will compile to the root directory
|
||||
# but their .cc source file will be in one of the following subdirectories
|
||||
vpath %.cc db:table:util:leveldb_ee:leveldb_os
|
||||
$(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
# special case for c_test
|
||||
vpath %.c db
|
||||
$(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
db_bench: db/db_bench.o $(LIBRARY) $(TESTUTIL)
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< $(TESTUTIL) -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
|
||||
.PHONY: STATIC_OBJDIRS
|
||||
STATIC_OBJDIRS: \
|
||||
$(STATIC_OUTDIR)/db \
|
||||
$(STATIC_OUTDIR)/port \
|
||||
$(STATIC_OUTDIR)/table \
|
||||
$(STATIC_OUTDIR)/util \
|
||||
$(STATIC_OUTDIR)/helpers/memenv
|
||||
|
||||
db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBRARY) $(TESTUTIL)
|
||||
$(SHARED_OUTDIR):
|
||||
mkdir $@
|
||||
|
||||
db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBRARY) $(TESTUTIL)
|
||||
$(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR)
|
||||
mkdir -p $@
|
||||
|
||||
#
|
||||
# build line taken from lz4 makefile
|
||||
#
|
||||
util/lz4.o: util/lz4.c util/lz4.h
|
||||
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -O3 -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"r130\" -c util/lz4.c -o util/lz4.o
|
||||
$(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
#
|
||||
# memory env
|
||||
#
|
||||
$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(MEMENVOBJECTS)
|
||||
$(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
|
||||
$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LDFLAGS)
|
||||
$(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
.PHONY: SHARED_OBJDIRS
|
||||
SHARED_OBJDIRS: \
|
||||
$(SHARED_OUTDIR)/db \
|
||||
$(SHARED_OUTDIR)/port \
|
||||
$(SHARED_OUTDIR)/table \
|
||||
$(SHARED_OUTDIR)/util \
|
||||
$(SHARED_OUTDIR)/helpers/memenv
|
||||
|
||||
$(DEVICE_OUTDIR):
|
||||
mkdir $@
|
||||
|
||||
$(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR)
|
||||
mkdir -p $@
|
||||
|
||||
$(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
.PHONY: DEVICE_OBJDIRS
|
||||
DEVICE_OBJDIRS: \
|
||||
$(DEVICE_OUTDIR)/db \
|
||||
$(DEVICE_OUTDIR)/port \
|
||||
$(DEVICE_OUTDIR)/table \
|
||||
$(DEVICE_OUTDIR)/util \
|
||||
$(DEVICE_OUTDIR)/helpers/memenv
|
||||
|
||||
$(SIMULATOR_OUTDIR):
|
||||
mkdir $@
|
||||
|
||||
$(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR)
|
||||
mkdir -p $@
|
||||
|
||||
$(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
$(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR)
|
||||
mkdir $@
|
||||
|
||||
.PHONY: SIMULATOR_OBJDIRS
|
||||
SIMULATOR_OBJDIRS: \
|
||||
$(SIMULATOR_OUTDIR)/db \
|
||||
$(SIMULATOR_OUTDIR)/port \
|
||||
$(SIMULATOR_OUTDIR)/table \
|
||||
$(SIMULATOR_OUTDIR)/util \
|
||||
$(SIMULATOR_OUTDIR)/helpers/memenv
|
||||
|
||||
$(STATIC_ALLOBJS): | STATIC_OBJDIRS
|
||||
$(DEVICE_ALLOBJS): | DEVICE_OBJDIRS
|
||||
$(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS
|
||||
$(SHARED_ALLOBJS): | SHARED_OBJDIRS
|
||||
|
||||
#
|
||||
# IOS build
|
||||
#
|
||||
ifeq ($(PLATFORM), IOS)
|
||||
# For iOS, create universal object files to be used on both the simulator and
|
||||
$(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(DEVICE_LIBOBJECTS)
|
||||
|
||||
$(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(SIMULATOR_LIBOBJECTS)
|
||||
|
||||
$(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(DEVICE_MEMENVOBJECTS)
|
||||
|
||||
$(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS)
|
||||
|
||||
# For iOS, create universal object libraries to be used on both the simulator and
|
||||
# a device.
|
||||
PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms
|
||||
SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer
|
||||
DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer
|
||||
IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString)
|
||||
|
||||
.cc.o:
|
||||
mkdir -p ios-x86/$(dir $@)
|
||||
$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
|
||||
mkdir -p ios-arm/$(dir $@)
|
||||
$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
|
||||
lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
|
||||
.c.o:
|
||||
mkdir -p ios-x86/$(dir $@)
|
||||
$(SIMULATORROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
|
||||
mkdir -p ios-arm/$(dir $@)
|
||||
$(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
|
||||
lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
$(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a
|
||||
lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@
|
||||
|
||||
$(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a
|
||||
lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@
|
||||
else
|
||||
#
|
||||
# build for everything NOT IOS
|
||||
#
|
||||
.cc.o:
|
||||
$(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(STATIC_LIBOBJECTS)
|
||||
|
||||
$(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(STATIC_MEMENVOBJECTS)
|
||||
endif
|
||||
|
||||
$(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(SHARED_MEMENVOBJECTS)
|
||||
|
||||
$(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/env_posix_test:util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS)
|
||||
$(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
$(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL)
|
||||
$(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS)
|
||||
|
||||
.PHONY: run-shared
|
||||
run-shared: $(SHARED_OUTDIR)/db_bench
|
||||
LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench
|
||||
|
||||
$(SIMULATOR_OUTDIR)/%.o: %.cc
|
||||
xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
|
||||
|
||||
$(DEVICE_OUTDIR)/%.o: %.cc
|
||||
xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
|
||||
|
||||
$(SIMULATOR_OUTDIR)/%.o: %.c
|
||||
xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
|
||||
|
||||
$(DEVICE_OUTDIR)/%.o: %.c
|
||||
xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
|
||||
|
||||
$(STATIC_OUTDIR)/%.o: %.cc
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
$(STATIC_OUTDIR)/%.o: %.c
|
||||
$(CC) $(CFLAGS) -c $< -o $@
|
||||
|
||||
$(SHARED_OUTDIR)/%.o: %.cc
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
|
||||
|
||||
.c.o:
|
||||
$(SHARED_OUTDIR)/%.o: %.c
|
||||
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
|
||||
|
||||
## @echo -- Creating dependency file for $<
|
||||
%.d: %.cc
|
||||
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM -E -MT $(basename $@).d -MT $(basename $@).o -MF $@ $<
|
||||
@echo $(basename $@).o: $(basename $@).d >>$@
|
||||
$(STATIC_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@
|
||||
|
||||
# generic build for command line tests
|
||||
%: %.cc
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< $(TESTHARNESS) -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
|
||||
|
||||
%: db/%.c
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< $(TESTHARNESS) -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
|
||||
|
||||
# for tools, omits test harness
|
||||
%: tools/%.cc
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
|
||||
|
||||
endif
|
||||
|
||||
#
|
||||
# load dependency files
|
||||
#
|
||||
ifeq ($(filter tar clean allclean distclean,$(MAKECMDGOALS)),)
|
||||
-include $(DEPEND)
|
||||
endif
|
||||
$(SHARED_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@
|
||||
@@ -1,83 +0,0 @@
|
||||
leveldb: A key-value store
|
||||
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
|
||||
The original Google README is now README.GOOGLE.
|
||||
|
||||
** Introduction
|
||||
|
||||
This repository contains the Google source code as modified to benefit
|
||||
the Riak environment. The typical Riak environment has two attributes
|
||||
that necessitate leveldb adjustments, both in options and code:
|
||||
|
||||
- production servers: Riak often runs in heavy Internet environments:
|
||||
servers with many CPU cores, lots of memory, and 24x7 disk activity.
|
||||
Basho's leveldb takes advantage of the environment by adding
|
||||
hardware CRC calculation, increasing Bloom filter accuracy, and
|
||||
defaulting to integrity checking enabled.
|
||||
|
||||
- multiple databases open: Riak opens 8 to 128 databases
|
||||
simultaneously. Google's leveldb supports this, but its background
|
||||
compaction thread can fall behind. leveldb will "stall" new user
|
||||
writes whenever the compaction thread gets too far behind. Basho's
|
||||
leveldb modification include multiple thread blocks that each
|
||||
contain prioritized threads for specific compaction activities.
|
||||
|
||||
Details for Basho's customizations exist in the leveldb wiki:
|
||||
|
||||
http://github.com/basho/leveldb/wiki
|
||||
|
||||
|
||||
** Branch pattern
|
||||
|
||||
This repository follows the Basho standard for branch management
|
||||
as of November 28, 2013. The standard is found here:
|
||||
|
||||
https://github.com/basho/riak/wiki/Basho-repository-management
|
||||
|
||||
In summary, the "develop" branch contains the most recently reviewed
|
||||
engineering work. The "master" branch contains the most recently
|
||||
released work, i.e. distributed as part of a Riak release.
|
||||
|
||||
|
||||
** Basic options needed
|
||||
|
||||
Those wishing to truly savor the benefits of Basho's modifications
|
||||
need to initialize a new leveldb::Options structure similar to the
|
||||
following before each call to leveldb::DB::Open:
|
||||
|
||||
leveldb::Options * options;
|
||||
|
||||
options=new Leveldb::Options;
|
||||
|
||||
options.filter_policy=leveldb::NewBloomFilterPolicy2(16);
|
||||
options.write_buffer_size=62914560; // 60Mbytes
|
||||
options.total_leveldb_mem=2684354560; // 2.5Gbytes (details below)
|
||||
options.env=leveldb::Env::Default();
|
||||
|
||||
|
||||
** Memory plan
|
||||
|
||||
Basho's leveldb dramatically departed from Google's original internal
|
||||
memory allotment plan with Riak 2.0. Basho's leveldb uses a methodology
|
||||
called flexcache. The technical details are here:
|
||||
|
||||
https://github.com/basho/leveldb/wiki/mv-flexcache
|
||||
|
||||
The key points are:
|
||||
|
||||
- options.total_leveldb_mem is an allocation for the entire process,
|
||||
not a single database
|
||||
|
||||
- giving different values to options.total_leveldb_mem on subsequent Open
|
||||
calls causes memory to rearrange to current value across all databases
|
||||
|
||||
- recommended minimum for Basho's leveldb is 340Mbytes per database.
|
||||
|
||||
- performance improves rapidly from 340Mbytes to 2.5Gbytes per database (3.0Gbytes
|
||||
if using Riak's active anti-entropy). Even more is nice, but not as helpful.
|
||||
|
||||
- never assign more than 75% of available RAM to total_leveldb_mem. There is
|
||||
too much unaccounted memory overhead (worse if you use tcmalloc library).
|
||||
|
||||
- options.max_open_files and options.block_cache should not be used.
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
leveldb: A key-value store
|
||||
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
|
||||
The code under this directory implements a system for maintaining a
|
||||
persistent key/value store.
|
||||
|
||||
See doc/index.html for more explanation.
|
||||
See doc/impl.html for a brief overview of the implementation.
|
||||
|
||||
The public interface is in include/*.h. Callers should not include or
|
||||
rely on the details of any other header files in this package. Those
|
||||
internal APIs may be changed without warning.
|
||||
|
||||
Guide to header files:
|
||||
|
||||
include/db.h
|
||||
Main interface to the DB: Start here
|
||||
|
||||
include/options.h
|
||||
Control over the behavior of an entire database, and also
|
||||
control over the behavior of individual reads and writes.
|
||||
|
||||
include/comparator.h
|
||||
Abstraction for user-specified comparison function. If you want
|
||||
just bytewise comparison of keys, you can use the default comparator,
|
||||
but clients can write their own comparator implementations if they
|
||||
want custom ordering (e.g. to handle different character
|
||||
encodings, etc.)
|
||||
|
||||
include/iterator.h
|
||||
Interface for iterating over data. You can get an iterator
|
||||
from a DB object.
|
||||
|
||||
include/write_batch.h
|
||||
Interface for atomically applying multiple updates to a database.
|
||||
|
||||
include/slice.h
|
||||
A simple module for maintaining a pointer and a length into some
|
||||
other byte array.
|
||||
|
||||
include/status.h
|
||||
Status is returned from many of the public interfaces and is used
|
||||
to report success and various kinds of errors.
|
||||
|
||||
include/env.h
|
||||
Abstraction of the OS environment. A posix implementation of
|
||||
this interface is in util/env_posix.cc
|
||||
|
||||
include/table.h
|
||||
include/table_builder.h
|
||||
Lower-level modules that most clients probably won't use directly
|
||||
@@ -0,0 +1,174 @@
|
||||
**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
|
||||
|
||||
[](https://travis-ci.org/google/leveldb)
|
||||
|
||||
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
|
||||
# Features
|
||||
* Keys and values are arbitrary byte arrays.
|
||||
* Data is stored sorted by key.
|
||||
* Callers can provide a custom comparison function to override the sort order.
|
||||
* The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`.
|
||||
* Multiple changes can be made in one atomic batch.
|
||||
* Users can create a transient snapshot to get a consistent view of data.
|
||||
* Forward and backward iteration is supported over the data.
|
||||
* Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/).
|
||||
* External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
|
||||
|
||||
# Documentation
|
||||
[LevelDB library documentation](https://github.com/google/leveldb/blob/master/doc/index.md) is online and bundled with the source code.
|
||||
|
||||
|
||||
# Limitations
|
||||
* This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes.
|
||||
* Only a single process (possibly multi-threaded) can access a particular database at a time.
|
||||
* There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library.
|
||||
|
||||
# Contributing to the leveldb Project
|
||||
The leveldb project welcomes contributions. leveldb's primary goal is to be
|
||||
a reliable and fast key/value store. Changes that are in line with the
|
||||
features/limitations outlined above, and meet the requirements below,
|
||||
will be considered.
|
||||
|
||||
Contribution requirements:
|
||||
|
||||
1. **POSIX only**. We _generally_ will only accept changes that are both
|
||||
compiled, and tested on a POSIX platform - usually Linux. Very small
|
||||
changes will sometimes be accepted, but consider that more of an
|
||||
exception than the rule.
|
||||
|
||||
2. **Stable API**. We strive very hard to maintain a stable API. Changes that
|
||||
require changes for projects using leveldb _might_ be rejected without
|
||||
sufficient benefit to the project.
|
||||
|
||||
3. **Tests**: All changes must be accompanied by a new (or changed) test, or
|
||||
a sufficient explanation as to why a new (or changed) test is not required.
|
||||
|
||||
## Submitting a Pull Request
|
||||
Before any pull request will be accepted the author must first sign a
|
||||
Contributor License Agreement (CLA) at https://cla.developers.google.com/.
|
||||
|
||||
In order to keep the commit timeline linear
|
||||
[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits)
|
||||
your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase)
|
||||
on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed
|
||||
with the internal repository at Google. More information at GitHub's
|
||||
[About Git rebase](https://help.github.com/articles/about-git-rebase/) page.
|
||||
|
||||
# Performance
|
||||
|
||||
Here is a performance report (with explanations) from the run of the
|
||||
included db_bench program. The results are somewhat noisy, but should
|
||||
be enough to get a ballpark performance estimate.
|
||||
|
||||
## Setup
|
||||
|
||||
We use a database with a million entries. Each entry has a 16 byte
|
||||
key, and a 100 byte value. Values used by the benchmark compress to
|
||||
about half their original size.
|
||||
|
||||
LevelDB: version 1.1
|
||||
Date: Sun May 1 12:11:26 2011
|
||||
CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz
|
||||
CPUCache: 4096 KB
|
||||
Keys: 16 bytes each
|
||||
Values: 100 bytes each (50 bytes after compression)
|
||||
Entries: 1000000
|
||||
Raw Size: 110.6 MB (estimated)
|
||||
File Size: 62.9 MB (estimated)
|
||||
|
||||
## Write performance
|
||||
|
||||
The "fill" benchmarks create a brand new database, in either
|
||||
sequential, or random order. The "fillsync" benchmark flushes data
|
||||
from the operating system to the disk after every operation; the other
|
||||
write operations leave the data sitting in the operating system buffer
|
||||
cache for a while. The "overwrite" benchmark does random writes that
|
||||
update existing keys in the database.
|
||||
|
||||
fillseq : 1.765 micros/op; 62.7 MB/s
|
||||
fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops)
|
||||
fillrandom : 2.460 micros/op; 45.0 MB/s
|
||||
overwrite : 2.380 micros/op; 46.5 MB/s
|
||||
|
||||
Each "op" above corresponds to a write of a single key/value pair.
|
||||
I.e., a random write benchmark goes at approximately 400,000 writes per second.
|
||||
|
||||
Each "fillsync" operation costs much less (0.3 millisecond)
|
||||
than a disk seek (typically 10 milliseconds). We suspect that this is
|
||||
because the hard disk itself is buffering the update in its memory and
|
||||
responding before the data has been written to the platter. This may
|
||||
or may not be safe based on whether or not the hard disk has enough
|
||||
power to save its memory in the event of a power failure.
|
||||
|
||||
## Read performance
|
||||
|
||||
We list the performance of reading sequentially in both the forward
|
||||
and reverse direction, and also the performance of a random lookup.
|
||||
Note that the database created by the benchmark is quite small.
|
||||
Therefore the report characterizes the performance of leveldb when the
|
||||
working set fits in memory. The cost of reading a piece of data that
|
||||
is not present in the operating system buffer cache will be dominated
|
||||
by the one or two disk seeks needed to fetch the data from disk.
|
||||
Write performance will be mostly unaffected by whether or not the
|
||||
working set fits in memory.
|
||||
|
||||
readrandom : 16.677 micros/op; (approximately 60,000 reads per second)
|
||||
readseq : 0.476 micros/op; 232.3 MB/s
|
||||
readreverse : 0.724 micros/op; 152.9 MB/s
|
||||
|
||||
LevelDB compacts its underlying storage data in the background to
|
||||
improve read performance. The results listed above were done
|
||||
immediately after a lot of random writes. The results after
|
||||
compactions (which are usually triggered automatically) are better.
|
||||
|
||||
readrandom : 11.602 micros/op; (approximately 85,000 reads per second)
|
||||
readseq : 0.423 micros/op; 261.8 MB/s
|
||||
readreverse : 0.663 micros/op; 166.9 MB/s
|
||||
|
||||
Some of the high cost of reads comes from repeated decompression of blocks
|
||||
read from disk. If we supply enough cache to the leveldb so it can hold the
|
||||
uncompressed blocks in memory, the read performance improves again:
|
||||
|
||||
readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction)
|
||||
readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction)
|
||||
|
||||
## Repository contents
|
||||
|
||||
See [doc/index.md](doc/index.md) for more explanation. See
|
||||
[doc/impl.md](doc/impl.md) for a brief overview of the implementation.
|
||||
|
||||
The public interface is in include/*.h. Callers should not include or
|
||||
rely on the details of any other header files in this package. Those
|
||||
internal APIs may be changed without warning.
|
||||
|
||||
Guide to header files:
|
||||
|
||||
* **include/db.h**: Main interface to the DB: Start here
|
||||
|
||||
* **include/options.h**: Control over the behavior of an entire database,
|
||||
and also control over the behavior of individual reads and writes.
|
||||
|
||||
* **include/comparator.h**: Abstraction for user-specified comparison function.
|
||||
If you want just bytewise comparison of keys, you can use the default
|
||||
comparator, but clients can write their own comparator implementations if they
|
||||
want custom ordering (e.g. to handle different character encodings, etc.)
|
||||
|
||||
* **include/iterator.h**: Interface for iterating over data. You can get
|
||||
an iterator from a DB object.
|
||||
|
||||
* **include/write_batch.h**: Interface for atomically applying multiple
|
||||
updates to a database.
|
||||
|
||||
* **include/slice.h**: A simple module for maintaining a pointer and a
|
||||
length into some other byte array.
|
||||
|
||||
* **include/status.h**: Status is returned from many of the public interfaces
|
||||
and is used to report success and various kinds of errors.
|
||||
|
||||
* **include/env.h**:
|
||||
Abstraction of the OS environment. A posix implementation of this interface is
|
||||
in util/env_posix.cc
|
||||
|
||||
* **include/table.h, include/table_builder.h**: Lower-level modules that most
|
||||
clients probably won't use directly
|
||||
@@ -7,6 +7,7 @@ db
|
||||
within [start_key..end_key]? For Chrome, deletion of obsolete
|
||||
object stores, etc. can be done in the background anyway, so
|
||||
probably not that important.
|
||||
- There have been requests for MultiGet.
|
||||
|
||||
After a range is completely deleted, what gets rid of the
|
||||
corresponding files if we do no future changes to that range. Make
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Building LevelDB On Windows
|
||||
|
||||
## Prereqs
|
||||
|
||||
Install the [Windows Software Development Kit version 7.1](http://www.microsoft.com/downloads/dlx/en-us/listdetailsview.aspx?FamilyID=6b6c21d2-2006-4afa-9702-529fa782d63b).
|
||||
|
||||
Download and extract the [Snappy source distribution](http://snappy.googlecode.com/files/snappy-1.0.5.tar.gz)
|
||||
|
||||
1. Open the "Windows SDK 7.1 Command Prompt" :
|
||||
Start Menu -> "Microsoft Windows SDK v7.1" > "Windows SDK 7.1 Command Prompt"
|
||||
2. Change the directory to the leveldb project
|
||||
|
||||
## Building the Static lib
|
||||
|
||||
* 32 bit Version
|
||||
|
||||
setenv /x86
|
||||
msbuild.exe /p:Configuration=Release /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5
|
||||
|
||||
* 64 bit Version
|
||||
|
||||
setenv /x64
|
||||
msbuild.exe /p:Configuration=Release /p:Platform=x64 /p:Snappy=..\snappy-1.0.5
|
||||
|
||||
|
||||
## Building and Running the Benchmark app
|
||||
|
||||
* 32 bit Version
|
||||
|
||||
setenv /x86
|
||||
msbuild.exe /p:Configuration=Benchmark /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5
|
||||
Benchmark\leveldb.exe
|
||||
|
||||
* 64 bit Version
|
||||
|
||||
setenv /x64
|
||||
msbuild.exe /p:Configuration=Benchmark /p:Platform=x64 /p:Snappy=..\snappy-1.0.5
|
||||
x64\Benchmark\leveldb.exe
|
||||
|
||||
@@ -7,8 +7,11 @@
|
||||
# CC C Compiler path
|
||||
# CXX C++ Compiler path
|
||||
# PLATFORM_LDFLAGS Linker flags
|
||||
# PLATFORM_LIBS Libraries flags
|
||||
# PLATFORM_SHARED_EXT Extension for shared libraries
|
||||
# PLATFORM_SHARED_LDFLAGS Flags for building shared library
|
||||
# This flag is embedded just before the name
|
||||
# of the shared library without intervening spaces
|
||||
# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
|
||||
# PLATFORM_CCFLAGS C compiler flags
|
||||
# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
|
||||
@@ -17,14 +20,15 @@
|
||||
#
|
||||
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
|
||||
#
|
||||
# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present
|
||||
# -DLEVELDB_ATOMIC_PRESENT if <atomic> is present
|
||||
# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
|
||||
# -DSNAPPY if the Snappy library is present
|
||||
#
|
||||
|
||||
OUTPUT=$1
|
||||
if test -z "$OUTPUT"; then
|
||||
echo "usage: $0 <output-filename>" >&2
|
||||
PREFIX=$2
|
||||
if test -z "$OUTPUT" || test -z "$PREFIX"; then
|
||||
echo "usage: $0 <output-filename> <directory_prefix>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -40,6 +44,10 @@ if test -z "$CXX"; then
|
||||
CXX=g++
|
||||
fi
|
||||
|
||||
if test -z "$TMPDIR"; then
|
||||
TMPDIR=/tmp
|
||||
fi
|
||||
|
||||
# Detect OS
|
||||
if test -z "$TARGET_OS"; then
|
||||
TARGET_OS=`uname -s`
|
||||
@@ -50,77 +58,119 @@ CROSS_COMPILE=
|
||||
PLATFORM_CCFLAGS=
|
||||
PLATFORM_CXXFLAGS=
|
||||
PLATFORM_LDFLAGS=
|
||||
PLATFORM_SHARED_EXT=
|
||||
PLATFORM_LIBS=
|
||||
PLATFORM_SHARED_EXT="so"
|
||||
PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
|
||||
PLATFORM_SHARED_CFLAGS="-fPIC"
|
||||
PLATFORM_SHARED_VERSIONED=true
|
||||
PLATFORM_SSEFLAGS=
|
||||
|
||||
if test -n "$LEVELDB_VSN"; then
|
||||
VERSION_FLAGS="$VERSION_FLAGS -DLEVELDB_VSN=\"$LEVELDB_VSN\""
|
||||
MEMCMP_FLAG=
|
||||
if [ "$CXX" = "g++" ]; then
|
||||
# Use libc's memcmp instead of GCC's memcmp. This results in ~40%
|
||||
# performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
|
||||
MEMCMP_FLAG="-fno-builtin-memcmp"
|
||||
fi
|
||||
|
||||
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
|
||||
case "$TARGET_OS" in
|
||||
CYGWIN_*)
|
||||
PLATFORM=OS_LINUX
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN"
|
||||
PLATFORM_LDFLAGS="-lpthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
Darwin)
|
||||
PLATFORM=OS_MACOSX
|
||||
oIFS="$IFS"; IFS=.
|
||||
set `uname -r`
|
||||
IFS="$oIFS"
|
||||
if [ "$1" -ge 13 ]; then
|
||||
# assume clang compiler
|
||||
COMMON_FLAGS="-mmacosx-version-min=10.8 -DOS_MACOSX -stdlib=libc++"
|
||||
PLATFORM_LDFLAGS="-mmacosx-version-min=10.8"
|
||||
else
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -DOS_MACOSX"
|
||||
fi
|
||||
PLATFORM_SHARED_EXT=
|
||||
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
|
||||
PLATFORM_SHARED_EXT=dylib
|
||||
[ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
|
||||
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
Linux)
|
||||
PLATFORM=OS_LINUX
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -pthread -DOS_LINUX"
|
||||
PLATFORM_LDFLAGS="-pthread -lrt"
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
|
||||
PLATFORM_LDFLAGS="-pthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
SunOS)
|
||||
PLATFORM=OS_SOLARIS
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
|
||||
PLATFORM_LDFLAGS="-lpthread -lrt"
|
||||
PLATFORM_SHARED_EXT=
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
|
||||
PLATFORM_LIBS="-lpthread -lrt"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
FreeBSD)
|
||||
CC=cc
|
||||
CXX=c++
|
||||
PLATFORM=OS_FREEBSD
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
|
||||
PLATFORM_LDFLAGS="-lpthread"
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
|
||||
PLATFORM_LIBS="-lpthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
GNU/kFreeBSD)
|
||||
PLATFORM=OS_KFREEBSD
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_KFREEBSD"
|
||||
PLATFORM_LIBS="-lpthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
NetBSD)
|
||||
PLATFORM=OS_NETBSD
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
|
||||
PLATFORM_LDFLAGS="-lpthread -lgcc_s"
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
|
||||
PLATFORM_LIBS="-lpthread -lgcc_s"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
OpenBSD)
|
||||
PLATFORM=OS_OPENBSD
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
|
||||
PLATFORM_LDFLAGS="-pthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
DragonFly)
|
||||
PLATFORM=OS_DRAGONFLYBSD
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
|
||||
PLATFORM_LDFLAGS="-lpthread"
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
|
||||
PLATFORM_LIBS="-lpthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
;;
|
||||
OS_ANDROID_CROSSCOMPILE)
|
||||
PLATFORM=OS_ANDROID
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
|
||||
PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
CROSS_COMPILE=true
|
||||
;;
|
||||
HP-UX)
|
||||
PLATFORM=OS_HPUX
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
|
||||
PLATFORM_LDFLAGS="-pthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
# man ld: +h internal_name
|
||||
PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
|
||||
;;
|
||||
IOS)
|
||||
PLATFORM=IOS
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
|
||||
[ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
|
||||
PORT_FILE=port/port_posix.cc
|
||||
PORT_SSE_FILE=port/port_posix_sse.cc
|
||||
PLATFORM_SHARED_EXT=
|
||||
PLATFORM_SHARED_LDFLAGS=
|
||||
PLATFORM_SHARED_CFLAGS=
|
||||
PLATFORM_SHARED_VERSIONED=
|
||||
;;
|
||||
OS_WINDOWS_CROSSCOMPILE | NATIVE_WINDOWS)
|
||||
PLATFORM=OS_WINDOWS
|
||||
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1"
|
||||
PLATFORM_SOURCES="util/env_win.cc"
|
||||
PLATFORM_LIBS="-lshlwapi"
|
||||
PORT_FILE=port/port_win.cc
|
||||
CROSS_COMPILE=true
|
||||
;;
|
||||
*)
|
||||
@@ -132,106 +182,78 @@ esac
|
||||
# except for the test and benchmark files. By default, find will output a list
|
||||
# of all files matching either rule, so we need to append -print to make the
|
||||
# prune take effect.
|
||||
if [ -f leveldb_ee/README.md ]; then
|
||||
DIRS="util db table leveldb_ee"
|
||||
else
|
||||
DIRS="util db table leveldb_os"
|
||||
fi
|
||||
DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
|
||||
|
||||
set -f # temporarily disable globbing so that our patterns aren't expanded
|
||||
PRUNE_TEST="-name *test*.cc -prune"
|
||||
PRUNE_BENCH="-name *_bench.cc -prune"
|
||||
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
|
||||
TESTS=`find $DIRS -name '*_test.c*' -print | sort | tr "\n" " "`
|
||||
PRUNE_TOOL="-name leveldbutil.cc -prune"
|
||||
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
|
||||
|
||||
set +f # re-enable globbing
|
||||
|
||||
# The sources consist of the portable files, plus the platform-specific port
|
||||
# file.
|
||||
echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
|
||||
echo "SOURCES=$PORTABLE_FILES $PORT_FILE $PORT_SSE_FILE" >> $OUTPUT
|
||||
echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
|
||||
echo "TEST_SOURCES=$TESTS" >>$OUTPUT
|
||||
|
||||
if [ "$CROSS_COMPILE" = "true" ]; then
|
||||
# Cross-compiling; do not try any compilation tests.
|
||||
true
|
||||
else
|
||||
# If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h.
|
||||
$CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
|
||||
#include <cstdatomic>
|
||||
CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
|
||||
|
||||
# If -std=c++0x works, use <atomic> as fallback for when memory barriers
|
||||
# are not available.
|
||||
$CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
|
||||
#include <atomic>
|
||||
int main() {}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_CSTDATOMIC_PRESENT"
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT"
|
||||
PLATFORM_CXXFLAGS="-std=c++0x"
|
||||
else
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
|
||||
fi
|
||||
|
||||
# Test whether Snappy library is installed
|
||||
# http://code.google.com/p/snappy/
|
||||
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
||||
#include <snappy.h>
|
||||
# Test whether tcmalloc is available
|
||||
$CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null <<EOF
|
||||
int main() {}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
|
||||
if [ "$PLATFORM" = "OS_LINUX" ]; then
|
||||
# Basho: switching to static snappy library to make tools more portable
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -Wl,-non_shared -lsnappy -Wl,-call_shared"
|
||||
else
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
|
||||
fi
|
||||
PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
|
||||
fi
|
||||
|
||||
# Test whether tcmalloc is available
|
||||
$CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
|
||||
rm -f $CXXOUTPUT 2>/dev/null
|
||||
|
||||
# Test if gcc SSE 4.2 is supported
|
||||
$CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -msse4.2 2>/dev/null <<EOF
|
||||
int main() {}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
|
||||
PLATFORM_SSEFLAGS="-msse4.2"
|
||||
fi
|
||||
|
||||
rm -f $CXXOUTPUT 2>/dev/null
|
||||
fi
|
||||
|
||||
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS $VERSION_FLAGS"
|
||||
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS $VERSION_FLAGS"
|
||||
# Use the SSE 4.2 CRC32C intrinsics iff runtime checks indicate compiler supports them.
|
||||
if [ -n "$PLATFORM_SSEFLAGS" ]; then
|
||||
PLATFORM_SSEFLAGS="$PLATFORM_SSEFLAGS -DLEVELDB_PLATFORM_POSIX_SSE"
|
||||
fi
|
||||
|
||||
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
|
||||
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
|
||||
|
||||
echo "CC=$CC" >> $OUTPUT
|
||||
echo "CXX=$CXX" >> $OUTPUT
|
||||
echo "PLATFORM=$PLATFORM" >> $OUTPUT
|
||||
echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT
|
||||
echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SSEFLAGS=$PLATFORM_SSEFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
|
||||
|
||||
#
|
||||
# Basho extension to place -D variable in include/leveldb/ldb_config.h
|
||||
#
|
||||
|
||||
LDB_CONFIG="include/leveldb/ldb_config.h"
|
||||
|
||||
# Delete existing output, if it exists
|
||||
rm -f $LDB_CONFIG
|
||||
|
||||
write_config_h()
|
||||
{
|
||||
for param in $@
|
||||
do
|
||||
prefix=$(expr -- $param : "\(..\)")
|
||||
if [ X$prefix = "X-D" ]
|
||||
then
|
||||
echo "" >>$LDB_CONFIG
|
||||
echo "#ifndef $(expr -- $param : '..\(.*\)')" >>$LDB_CONFIG
|
||||
echo " #define $(expr -- $param : '..\(.*\)')" >>$LDB_CONFIG
|
||||
echo "#endif" >>$LDB_CONFIG
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
echo "/** This file is generated by build_detect_platform." >$LDB_CONFIG
|
||||
echo " * It saves the state of compile flags. This benefits the reuse" >>$LDB_CONFIG
|
||||
echo " * of internal include files outside of a leveldb build." >>$LDB_CONFIG
|
||||
echo " */" >>$LDB_CONFIG
|
||||
|
||||
write_config_h $COMMON_FLAGS
|
||||
@@ -0,0 +1,118 @@
|
||||
// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/db.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class AutoCompactTest {
|
||||
public:
|
||||
std::string dbname_;
|
||||
Cache* tiny_cache_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
|
||||
AutoCompactTest() {
|
||||
dbname_ = test::TmpDir() + "/autocompact_test";
|
||||
tiny_cache_ = NewLRUCache(100);
|
||||
options_.block_cache = tiny_cache_;
|
||||
DestroyDB(dbname_, options_);
|
||||
options_.create_if_missing = true;
|
||||
options_.compression = kNoCompression;
|
||||
ASSERT_OK(DB::Open(options_, dbname_, &db_));
|
||||
}
|
||||
|
||||
~AutoCompactTest() {
|
||||
delete db_;
|
||||
DestroyDB(dbname_, Options());
|
||||
delete tiny_cache_;
|
||||
}
|
||||
|
||||
std::string Key(int i) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "key%06d", i);
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
uint64_t Size(const Slice& start, const Slice& limit) {
|
||||
Range r(start, limit);
|
||||
uint64_t size;
|
||||
db_->GetApproximateSizes(&r, 1, &size);
|
||||
return size;
|
||||
}
|
||||
|
||||
void DoReads(int n);
|
||||
};
|
||||
|
||||
static const int kValueSize = 200 * 1024;
|
||||
static const int kTotalSize = 100 * 1024 * 1024;
|
||||
static const int kCount = kTotalSize / kValueSize;
|
||||
|
||||
// Read through the first n keys repeatedly and check that they get
|
||||
// compacted (verified by checking the size of the key space).
|
||||
void AutoCompactTest::DoReads(int n) {
|
||||
std::string value(kValueSize, 'x');
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
|
||||
// Fill database
|
||||
for (int i = 0; i < kCount; i++) {
|
||||
ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
|
||||
}
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
// Delete everything
|
||||
for (int i = 0; i < kCount; i++) {
|
||||
ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
|
||||
}
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
// Get initial measurement of the space we will be reading.
|
||||
const int64_t initial_size = Size(Key(0), Key(n));
|
||||
const int64_t initial_other_size = Size(Key(n), Key(kCount));
|
||||
|
||||
// Read until size drops significantly.
|
||||
std::string limit_key = Key(n);
|
||||
for (int read = 0; true; read++) {
|
||||
ASSERT_LT(read, 100) << "Taking too long to compact";
|
||||
Iterator* iter = db_->NewIterator(ReadOptions());
|
||||
for (iter->SeekToFirst();
|
||||
iter->Valid() && iter->key().ToString() < limit_key;
|
||||
iter->Next()) {
|
||||
// Drop data
|
||||
}
|
||||
delete iter;
|
||||
// Wait a little bit to allow any triggered compactions to complete.
|
||||
Env::Default()->SleepForMicroseconds(1000000);
|
||||
uint64_t size = Size(Key(0), Key(n));
|
||||
fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
|
||||
read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
|
||||
if (size <= initial_size/10) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that the size of the key space not touched by the reads
|
||||
// is pretty much unchanged.
|
||||
const int64_t final_other_size = Size(Key(n), Key(kCount));
|
||||
ASSERT_LE(final_other_size, initial_other_size + 1048576);
|
||||
ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
|
||||
}
|
||||
|
||||
TEST(AutoCompactTest, ReadAll) {
|
||||
DoReads(kCount);
|
||||
}
|
||||
|
||||
TEST(AutoCompactTest, ReadHalf) {
|
||||
DoReads(kCount/2);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
@@ -2,16 +2,12 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#include <inttypes.h>
|
||||
|
||||
#include "db/builder.h"
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "db/version_set.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
@@ -21,51 +17,27 @@ namespace leveldb {
|
||||
Status BuildTable(const std::string& dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
const Comparator * user_comparator,
|
||||
TableCache* table_cache,
|
||||
Iterator* iter,
|
||||
FileMetaData* meta,
|
||||
SequenceNumber smallest_snapshot) {
|
||||
FileMetaData* meta) {
|
||||
Status s;
|
||||
size_t keys_seen, keys_retired;
|
||||
|
||||
keys_seen=0;
|
||||
keys_retired=0;
|
||||
|
||||
meta->file_size = 0;
|
||||
iter->SeekToFirst();
|
||||
|
||||
KeyRetirement retire(user_comparator, smallest_snapshot, &options);
|
||||
|
||||
std::string fname = TableFileName(options, meta->number, meta->level);
|
||||
std::string fname = TableFileName(dbname, meta->number);
|
||||
if (iter->Valid()) {
|
||||
WritableFile* file;
|
||||
|
||||
s = env->NewWritableFile(fname, &file,
|
||||
env->RecoveryMmapSize(&options));
|
||||
s = env->NewWritableFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// tune fadvise to keep all of this lower level file in page cache
|
||||
// (compaction of unsorted files causes severe cache misses)
|
||||
file->SetMetadataOffset(1);
|
||||
|
||||
TableBuilder* builder = new TableBuilder(options, file);
|
||||
meta->smallest.DecodeFrom(iter->key());
|
||||
for (; iter->Valid(); iter->Next()) {
|
||||
++keys_seen;
|
||||
Slice key = iter->key();
|
||||
if (!retire(key))
|
||||
{
|
||||
meta->largest.DecodeFrom(key);
|
||||
builder->Add(key, iter->value());
|
||||
++meta->num_entries;
|
||||
} // if
|
||||
else
|
||||
{
|
||||
++keys_retired;
|
||||
} // else
|
||||
meta->largest.DecodeFrom(key);
|
||||
builder->Add(key, iter->value());
|
||||
}
|
||||
|
||||
// Finish and check for builder errors
|
||||
@@ -73,9 +45,6 @@ Status BuildTable(const std::string& dbname,
|
||||
s = builder->Finish();
|
||||
if (s.ok()) {
|
||||
meta->file_size = builder->FileSize();
|
||||
meta->exp_write_low = builder->GetExpiryWriteLow();
|
||||
meta->exp_write_high = builder->GetExpiryWriteHigh();
|
||||
meta->exp_explicit_high = builder->GetExpiryExplicitHigh();
|
||||
assert(meta->file_size > 0);
|
||||
}
|
||||
} else {
|
||||
@@ -95,20 +64,10 @@ Status BuildTable(const std::string& dbname,
|
||||
|
||||
if (s.ok()) {
|
||||
// Verify that the table is usable
|
||||
Table * table_ptr;
|
||||
Iterator* it = table_cache->NewIterator(ReadOptions(),
|
||||
meta->number,
|
||||
meta->file_size,
|
||||
meta->level,
|
||||
&table_ptr);
|
||||
meta->file_size);
|
||||
s = it->status();
|
||||
|
||||
// Riak specific: bloom filter is no longer read by default,
|
||||
// force read on highly used overlapped table files
|
||||
if (s.ok() && VersionSet::IsLevelOverlapped(meta->level))
|
||||
table_ptr->ReadFilter();
|
||||
|
||||
// table_ptr is owned by it and therefore invalidated by this delete
|
||||
delete it;
|
||||
}
|
||||
}
|
||||
@@ -120,11 +79,6 @@ Status BuildTable(const std::string& dbname,
|
||||
|
||||
if (s.ok() && meta->file_size > 0) {
|
||||
// Keep it
|
||||
if (0!=keys_retired)
|
||||
{
|
||||
Log(options.info_log, "Level-0 table #%" PRIu64 ": %zd keys seen, %zd keys retired, %zd keys expired",
|
||||
meta->number, keys_seen, retire.GetDroppedCount(), retire.GetExpiredCount());
|
||||
} // if
|
||||
} else {
|
||||
env->DeleteFile(fname);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#define STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
|
||||
#include "leveldb/status.h"
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@@ -26,11 +25,9 @@ class VersionEdit;
|
||||
extern Status BuildTable(const std::string& dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
const Comparator * user_comparator,
|
||||
TableCache* table_cache,
|
||||
Iterator* iter,
|
||||
FileMetaData* meta,
|
||||
SequenceNumber smallest_snapshot);
|
||||
FileMetaData* meta);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
||||
+8
-77
@@ -6,7 +6,6 @@
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <stdint.h>
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
@@ -41,8 +40,6 @@ using leveldb::Status;
|
||||
using leveldb::WritableFile;
|
||||
using leveldb::WriteBatch;
|
||||
using leveldb::WriteOptions;
|
||||
using leveldb::KeyMetaData;
|
||||
using leveldb::ValueType;
|
||||
|
||||
extern "C" {
|
||||
|
||||
@@ -52,7 +49,6 @@ struct leveldb_writebatch_t { WriteBatch rep; };
|
||||
struct leveldb_snapshot_t { const Snapshot* rep; };
|
||||
struct leveldb_readoptions_t { ReadOptions rep; };
|
||||
struct leveldb_writeoptions_t { WriteOptions rep; };
|
||||
struct leveldb_keymetadata_t { KeyMetaData rep; };
|
||||
struct leveldb_options_t { Options rep; };
|
||||
struct leveldb_cache_t { Cache* rep; };
|
||||
struct leveldb_seqfile_t { SequentialFile* rep; };
|
||||
@@ -177,19 +173,8 @@ void leveldb_put(
|
||||
const char* key, size_t keylen,
|
||||
const char* val, size_t vallen,
|
||||
char** errptr) {
|
||||
return(leveldb_put2(db, options, key, keylen, val, vallen, errptr, NULL));
|
||||
}
|
||||
|
||||
void leveldb_put2(
|
||||
leveldb_t* db,
|
||||
const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen,
|
||||
const char* val, size_t vallen,
|
||||
char** errptr,
|
||||
const leveldb_keymetadata_t * metadata) {
|
||||
SaveError(errptr,
|
||||
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen),
|
||||
(NULL==metadata ? NULL : &metadata->rep)));
|
||||
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
|
||||
}
|
||||
|
||||
void leveldb_delete(
|
||||
@@ -215,21 +200,9 @@ char* leveldb_get(
|
||||
const char* key, size_t keylen,
|
||||
size_t* vallen,
|
||||
char** errptr) {
|
||||
|
||||
return(leveldb_get2(db, options, key, keylen, vallen, errptr, NULL));
|
||||
}
|
||||
|
||||
char* leveldb_get2(
|
||||
leveldb_t* db,
|
||||
const leveldb_readoptions_t* options,
|
||||
const char* key, size_t keylen,
|
||||
size_t* vallen,
|
||||
char** errptr,
|
||||
leveldb_keymetadata_t * metadata) {
|
||||
char* result = NULL;
|
||||
std::string tmp;
|
||||
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp,
|
||||
(NULL==metadata ? NULL : &metadata->rep));
|
||||
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
|
||||
if (s.ok()) {
|
||||
*vallen = tmp.size();
|
||||
result = CopyString(tmp);
|
||||
@@ -357,15 +330,6 @@ const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) {
|
||||
return s.data();
|
||||
}
|
||||
|
||||
const void leveldb_iter_keymetadata(const leveldb_iterator_t* iter,
|
||||
leveldb_keymetadata_t * meta)
|
||||
{
|
||||
if (NULL!=iter && NULL!=meta)
|
||||
{
|
||||
meta->rep=iter->rep->keymetadata();
|
||||
} // if
|
||||
}
|
||||
|
||||
void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) {
|
||||
SaveError(errptr, iter->rep->status());
|
||||
}
|
||||
@@ -386,16 +350,7 @@ void leveldb_writebatch_put(
|
||||
leveldb_writebatch_t* b,
|
||||
const char* key, size_t klen,
|
||||
const char* val, size_t vlen) {
|
||||
leveldb_writebatch_put2(b, key, klen, val, vlen,NULL);
|
||||
}
|
||||
|
||||
void leveldb_writebatch_put2(
|
||||
leveldb_writebatch_t* b,
|
||||
const char* key, size_t klen,
|
||||
const char* val, size_t vlen,
|
||||
const leveldb_keymetadata_t * metadata) {
|
||||
b->rep.Put(Slice(key, klen), Slice(val, vlen),
|
||||
(NULL==metadata ? NULL : &metadata->rep));
|
||||
b->rep.Put(Slice(key, klen), Slice(val, vlen));
|
||||
}
|
||||
|
||||
void leveldb_writebatch_delete(
|
||||
@@ -407,20 +362,15 @@ void leveldb_writebatch_delete(
|
||||
void leveldb_writebatch_iterate(
|
||||
leveldb_writebatch_t* b,
|
||||
void* state,
|
||||
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen,
|
||||
const int & type, const uint64_t & expiry),
|
||||
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
|
||||
void (*deleted)(void*, const char* k, size_t klen)) {
|
||||
class H : public WriteBatch::Handler {
|
||||
public:
|
||||
void* state_;
|
||||
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen,
|
||||
const int & type, const uint64_t & expiry);
|
||||
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
|
||||
void (*deleted_)(void*, const char* k, size_t klen);
|
||||
virtual void Put(const Slice& key, const Slice& value,
|
||||
const leveldb::ValueType & type,
|
||||
const leveldb::ExpiryTimeMicros & expiry)
|
||||
{
|
||||
(*put_)(state_, key.data(), key.size(), value.data(), value.size(), (int)type, (uint64_t)expiry);
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
(*put_)(state_, key.data(), key.size(), value.data(), value.size());
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
(*deleted_)(state_, key.data(), key.size());
|
||||
@@ -468,11 +418,6 @@ void leveldb_options_set_paranoid_checks(
|
||||
opt->rep.paranoid_checks = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_verify_compactions(
|
||||
leveldb_options_t* opt, unsigned char v) {
|
||||
opt->rep.verify_compactions = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
|
||||
opt->rep.env = (env ? env->rep : NULL);
|
||||
}
|
||||
@@ -505,10 +450,6 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
|
||||
opt->rep.compression = static_cast<CompressionType>(t);
|
||||
}
|
||||
|
||||
void leveldb_options_set_total_leveldb_mem(leveldb_options_t* opt, size_t s) {
|
||||
opt->rep.total_leveldb_mem = s;
|
||||
}
|
||||
|
||||
leveldb_comparator_t* leveldb_comparator_create(
|
||||
void* state,
|
||||
void (*destructor)(void*),
|
||||
@@ -639,18 +580,8 @@ void leveldb_env_destroy(leveldb_env_t* env) {
|
||||
delete env;
|
||||
}
|
||||
|
||||
void leveldb_env_shutdown() {
|
||||
Env::Shutdown();
|
||||
}
|
||||
|
||||
/**
|
||||
* CAUTION: this call is only for char * objects returned by
|
||||
* functions like leveldb_get and leveldb_property_value.
|
||||
* Also used to release errptr strings.
|
||||
*/
|
||||
void leveldb_free(void* ptr) {
|
||||
if (NULL!=ptr)
|
||||
free(ptr);
|
||||
free(ptr);
|
||||
}
|
||||
|
||||
int leveldb_major_version() {
|
||||
|
||||
+12
-225
@@ -3,8 +3,6 @@
|
||||
found in the LICENSE file. See the AUTHORS file for names of contributors. */
|
||||
|
||||
#include "leveldb/c.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "port/port.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
@@ -13,13 +11,8 @@
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
using leveldb::ValueType;
|
||||
|
||||
struct leveldb_keymetadata_t { leveldb::KeyMetaData rep; };
|
||||
|
||||
const char* phase = "";
|
||||
static char dbname[200];
|
||||
static leveldb::ExpiryTimeMicros gStartTime;
|
||||
|
||||
static void StartPhase(const char* name) {
|
||||
fprintf(stderr, "=== Test %s\n", name);
|
||||
@@ -40,7 +33,7 @@ static const char* GetTempDir(void) {
|
||||
}
|
||||
|
||||
#define CheckCondition(cond) \
|
||||
if (!(cond)) { \
|
||||
if (!(cond)) { \
|
||||
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
|
||||
abort(); \
|
||||
}
|
||||
@@ -56,7 +49,7 @@ static void CheckEqual(const char* expected, const char* v, size_t n) {
|
||||
fprintf(stderr, "%s: expected '%s', got '%s'\n",
|
||||
phase,
|
||||
(expected ? expected : "(null)"),
|
||||
(v ? v : "(null)"));
|
||||
(v ? v : "(null"));
|
||||
abort();
|
||||
}
|
||||
}
|
||||
@@ -119,117 +112,6 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
|
||||
(*state)++;
|
||||
}
|
||||
|
||||
// (expiry enabled)
|
||||
static void CheckGet2(
|
||||
leveldb_t* db,
|
||||
const leveldb_readoptions_t* options,
|
||||
const char* key,
|
||||
const char* expected,
|
||||
ValueType type,
|
||||
uint64_t expiry) {
|
||||
char* err = NULL;
|
||||
size_t val_len;
|
||||
char* val;
|
||||
leveldb_keymetadata_t meta;
|
||||
|
||||
val = leveldb_get2(db, options, key, strlen(key), &val_len, &err, &meta);
|
||||
CheckNoError(err);
|
||||
CheckEqual(expected, val, val_len);
|
||||
CheckCondition(type==meta.rep.m_Type);
|
||||
if (0==expiry && leveldb::kTypeValueWriteTime==type)
|
||||
{
|
||||
leveldb::ExpiryTimeMicros now=leveldb::port::TimeMicros();
|
||||
CheckCondition(gStartTime<=meta.rep.m_Expiry && meta.rep.m_Expiry<=now);
|
||||
} // if
|
||||
else
|
||||
{CheckCondition(expiry==meta.rep.m_Expiry);}
|
||||
|
||||
Free(&val);
|
||||
}
|
||||
|
||||
// (expiry enabled)
|
||||
static void CheckIter2(leveldb_iterator_t* iter,
|
||||
const char* key, const char* val,
|
||||
const leveldb::KeyMetaData & meta) {
|
||||
size_t len;
|
||||
const char* str;
|
||||
leveldb_keymetadata_t it_meta;
|
||||
|
||||
str = leveldb_iter_key(iter, &len);
|
||||
CheckEqual(key, str, len);
|
||||
str = leveldb_iter_value(iter, &len);
|
||||
CheckEqual(val, str, len);
|
||||
|
||||
leveldb_iter_keymetadata(iter, &it_meta);
|
||||
CheckCondition(meta.m_Type==it_meta.rep.m_Type);
|
||||
if (0==meta.m_Expiry && leveldb::kTypeValueWriteTime==meta.m_Type)
|
||||
{
|
||||
leveldb::ExpiryTimeMicros now=leveldb::port::TimeMicros();
|
||||
CheckCondition(gStartTime<=it_meta.rep.m_Expiry && it_meta.rep.m_Expiry<=now);
|
||||
} // if
|
||||
else
|
||||
{CheckCondition(meta.m_Expiry==it_meta.rep.m_Expiry);}
|
||||
|
||||
}
|
||||
|
||||
// Callback from leveldb_writebatch_iterate()
|
||||
// (expiry enabled)
|
||||
struct CheckPut2Data
|
||||
{
|
||||
const char * m_Key;
|
||||
const char * m_Value;
|
||||
ValueType m_Type;
|
||||
uint64_t m_Expiry;
|
||||
};
|
||||
|
||||
static struct CheckPut2Data gCheckPut2Data[]=
|
||||
{
|
||||
{"foo","hello_put2",leveldb::kTypeValue,0},
|
||||
{"box","c_put2",leveldb::kTypeValue,0},
|
||||
{"disney","cartoon_put2",leveldb::kTypeValueWriteTime, 0},
|
||||
{"money","lotsof_put2",leveldb::kTypeValueWriteTime, 9988776655},
|
||||
{"time","ismoney_put2",leveldb::kTypeValueExplicitExpiry, 221199887766}
|
||||
};
|
||||
|
||||
static struct CheckPut2Data gCheckPut2ItrData[]=
|
||||
{
|
||||
{"bar","b",leveldb::kTypeValue,0},
|
||||
{"box","c",leveldb::kTypeValue,0},
|
||||
{"bar","",leveldb::kTypeDeletion,0},
|
||||
{"mom","texas",leveldb::kTypeValueWriteTime,0},
|
||||
{"dad","poland",leveldb::kTypeValueExplicitExpiry,22446688}
|
||||
};
|
||||
|
||||
static void CheckPut2(void* ptr,
|
||||
const char* k, size_t klen,
|
||||
const char* v, size_t vlen,
|
||||
const int & type_int,
|
||||
const uint64_t & expiry) {
|
||||
int* state = (int*) ptr;
|
||||
CheckCondition(*state < (sizeof(gCheckPut2ItrData)/sizeof(gCheckPut2ItrData[0])));
|
||||
struct CheckPut2Data * test;
|
||||
|
||||
test=&gCheckPut2ItrData[*state];
|
||||
CheckEqual(test->m_Key, k, klen);
|
||||
CheckEqual(test->m_Value, v, vlen);
|
||||
CheckCondition((int)test->m_Type==type_int);
|
||||
if (leveldb::kTypeValueWriteTime!=test->m_Type)
|
||||
{CheckCondition((uint64_t)test->m_Expiry==expiry);}
|
||||
(*state)++;
|
||||
}
|
||||
|
||||
// Callback from leveldb_writebatch_iterate()
|
||||
// (expiry enabled)
|
||||
static void CheckDel2(void* ptr, const char* k, size_t klen) {
|
||||
int* state = (int*) ptr;
|
||||
CheckCondition(*state < (sizeof(gCheckPut2ItrData)/sizeof(gCheckPut2ItrData[0])));
|
||||
struct CheckPut2Data * test;
|
||||
|
||||
test=&gCheckPut2ItrData[*state];
|
||||
CheckEqual(test->m_Key, k, klen);
|
||||
(*state)++;
|
||||
}
|
||||
|
||||
static void CmpDestroy(void* arg) { }
|
||||
|
||||
static int CmpCompare(void* arg, const char* a, size_t alen,
|
||||
@@ -259,7 +141,7 @@ static char* FilterCreate(
|
||||
int num_keys,
|
||||
size_t* filter_length) {
|
||||
*filter_length = 4;
|
||||
char* result = (char*)malloc(4);
|
||||
char* result = malloc(4);
|
||||
memcpy(result, "fake", 4);
|
||||
return result;
|
||||
}
|
||||
@@ -285,7 +167,6 @@ int main(int argc, char** argv) {
|
||||
|
||||
CheckCondition(leveldb_major_version() >= 1);
|
||||
CheckCondition(leveldb_minor_version() >= 1);
|
||||
gStartTime=leveldb::port::TimeMicros();
|
||||
|
||||
snprintf(dbname, sizeof(dbname),
|
||||
"%s/leveldb_c_test-%d",
|
||||
@@ -326,6 +207,12 @@ int main(int argc, char** argv) {
|
||||
CheckCondition(err != NULL);
|
||||
Free(&err);
|
||||
|
||||
StartPhase("leveldb_free");
|
||||
db = leveldb_open(options, dbname, &err);
|
||||
CheckCondition(err != NULL);
|
||||
leveldb_free(err);
|
||||
err = NULL;
|
||||
|
||||
StartPhase("open");
|
||||
leveldb_options_set_create_if_missing(options, 1);
|
||||
db = leveldb_open(options, dbname, &err);
|
||||
@@ -347,74 +234,42 @@ int main(int argc, char** argv) {
|
||||
|
||||
StartPhase("writebatch");
|
||||
{
|
||||
leveldb_keymetadata_t meta;
|
||||
leveldb_writebatch_t* wb = leveldb_writebatch_create();
|
||||
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
|
||||
leveldb_writebatch_clear(wb);
|
||||
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
|
||||
leveldb_writebatch_put(wb, "box", 3, "c", 1);
|
||||
leveldb_writebatch_delete(wb, "bar", 3);
|
||||
meta.rep.m_Type=leveldb::kTypeValueWriteTime;
|
||||
meta.rep.m_Expiry=0;
|
||||
leveldb_writebatch_put2(wb, "mom", 3, "texas", 5, &meta);
|
||||
meta.rep.m_Type=leveldb::kTypeValueExplicitExpiry;
|
||||
meta.rep.m_Expiry=22446688;
|
||||
leveldb_writebatch_put2(wb, "dad", 3, "poland", 6, &meta);
|
||||
leveldb_write(db, woptions, wb, &err);
|
||||
CheckNoError(err);
|
||||
CheckGet(db, roptions, "foo", "hello");
|
||||
CheckGet(db, roptions, "bar", NULL);
|
||||
CheckGet(db, roptions, "box", "c");
|
||||
CheckGet2(db, roptions, "dad", "poland", leveldb::kTypeValueExplicitExpiry, 22446688);
|
||||
CheckGet2(db, roptions, "mom", "texas", leveldb::kTypeValueWriteTime, 0);
|
||||
int pos = 0;
|
||||
leveldb_writebatch_iterate(wb, &pos, CheckPut2, CheckDel2);
|
||||
CheckCondition(pos == 5);
|
||||
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
|
||||
CheckCondition(pos == 3);
|
||||
leveldb_writebatch_destroy(wb);
|
||||
}
|
||||
|
||||
// reminder: keymetadata not supported on backward iteration
|
||||
StartPhase("iter");
|
||||
{
|
||||
leveldb::KeyMetaData meta;
|
||||
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
|
||||
CheckCondition(!leveldb_iter_valid(iter));
|
||||
leveldb_iter_seek_to_first(iter);
|
||||
CheckCondition(leveldb_iter_valid(iter));
|
||||
CheckIter(iter, "box", "c");
|
||||
meta.m_Type=leveldb::kTypeValue;
|
||||
meta.m_Expiry=0;
|
||||
CheckIter2(iter, "box", "c", meta);
|
||||
|
||||
meta.m_Type=leveldb::kTypeValueExplicitExpiry;
|
||||
meta.m_Expiry=22446688;
|
||||
leveldb_iter_next(iter);
|
||||
CheckIter2(iter, "dad", "poland", meta);
|
||||
leveldb_iter_next(iter);
|
||||
CheckIter(iter, "foo", "hello");
|
||||
leveldb_iter_prev(iter);
|
||||
CheckIter(iter, "dad", "poland");
|
||||
leveldb_iter_prev(iter);
|
||||
CheckIter(iter, "box", "c");
|
||||
leveldb_iter_prev(iter);
|
||||
CheckCondition(!leveldb_iter_valid(iter));
|
||||
leveldb_iter_seek_to_last(iter);
|
||||
CheckIter(iter, "mom", "texas");
|
||||
CheckIter(iter, "foo", "hello");
|
||||
leveldb_iter_seek(iter, "b", 1);
|
||||
CheckIter(iter, "box", "c");
|
||||
leveldb_iter_get_error(iter, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
meta.m_Type=leveldb::kTypeValue;
|
||||
meta.m_Expiry=0;
|
||||
CheckIter2(iter, "box", "c", meta);
|
||||
leveldb_iter_seek(iter, "m", 1);
|
||||
meta.m_Type=leveldb::kTypeValueWriteTime;
|
||||
meta.m_Expiry=0;
|
||||
CheckIter2(iter, "mom", "texas", meta);
|
||||
leveldb_iter_get_error(iter, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
leveldb_iter_destroy(iter);
|
||||
}
|
||||
|
||||
@@ -480,70 +335,6 @@ int main(int argc, char** argv) {
|
||||
leveldb_options_set_error_if_exists(options, 1);
|
||||
}
|
||||
|
||||
StartPhase("put expiry");
|
||||
{
|
||||
leveldb_keymetadata_t meta;
|
||||
int loop, count;
|
||||
|
||||
count = sizeof(gCheckPut2Data) / sizeof(gCheckPut2Data[0]);
|
||||
|
||||
for (loop=0; loop<count; ++loop)
|
||||
{
|
||||
size_t klen, vlen;
|
||||
leveldb_keymetadata_t meta;
|
||||
struct CheckPut2Data * test;
|
||||
|
||||
test=&gCheckPut2Data[loop];
|
||||
klen=strlen(test->m_Key);
|
||||
vlen=strlen(test->m_Value);
|
||||
meta.rep.m_Type=test->m_Type;
|
||||
meta.rep.m_Expiry=test->m_Expiry;
|
||||
|
||||
leveldb_put2(db, woptions, test->m_Key, klen,
|
||||
test->m_Value, vlen, &err,
|
||||
&meta);
|
||||
CheckNoError(err);
|
||||
} // for
|
||||
|
||||
// testing memtable right now
|
||||
for (loop=0; loop<count; ++loop)
|
||||
{
|
||||
size_t klen, vlen;
|
||||
leveldb_keymetadata_t meta;
|
||||
struct CheckPut2Data * test;
|
||||
|
||||
test=&gCheckPut2Data[loop];
|
||||
klen=strlen(test->m_Key);
|
||||
vlen=strlen(test->m_Value);
|
||||
|
||||
CheckGet2(db, roptions, test->m_Key, test->m_Value,
|
||||
test->m_Type, test->m_Expiry);
|
||||
} // for
|
||||
|
||||
// close and open to force memory table into .sst upon open
|
||||
leveldb_close(db);
|
||||
leveldb_options_set_error_if_exists(options, 0);
|
||||
db = leveldb_open(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
// now testing get from a level-0 .sst file
|
||||
for (loop=0; loop<count; ++loop)
|
||||
{
|
||||
size_t klen, vlen;
|
||||
leveldb_keymetadata_t meta;
|
||||
struct CheckPut2Data * test;
|
||||
|
||||
test=&gCheckPut2Data[loop];
|
||||
klen=strlen(test->m_Key);
|
||||
vlen=strlen(test->m_Value);
|
||||
|
||||
CheckGet2(db, roptions, test->m_Key, test->m_Value,
|
||||
test->m_Type, test->m_Expiry);
|
||||
} // for
|
||||
}
|
||||
|
||||
//
|
||||
// This screws up "options" for real database work. execute last.
|
||||
StartPhase("filter");
|
||||
for (run = 0; run < 2; run++) {
|
||||
// First run uses custom filter, second run uses bloom filter
|
||||
@@ -585,8 +376,6 @@ int main(int argc, char** argv) {
|
||||
leveldb_filterpolicy_destroy(policy);
|
||||
}
|
||||
|
||||
|
||||
|
||||
StartPhase("cleanup");
|
||||
leveldb_close(db);
|
||||
leveldb_options_destroy(options);
|
||||
@@ -597,7 +386,5 @@ int main(int argc, char** argv) {
|
||||
leveldb_env_destroy(env);
|
||||
|
||||
fprintf(stderr, "PASS\n");
|
||||
|
||||
leveldb_env_shutdown();
|
||||
return 0;
|
||||
}
|
||||
@@ -35,8 +35,8 @@ class CorruptionTest {
|
||||
CorruptionTest() {
|
||||
tiny_cache_ = NewLRUCache(100);
|
||||
options_.env = &env_;
|
||||
dbname_ = test::TmpDir() + "/db_test";
|
||||
dbname_ = MakeTieredDbname(dbname_, options_);
|
||||
options_.block_cache = tiny_cache_;
|
||||
dbname_ = test::TmpDir() + "/corruption_test";
|
||||
DestroyDB(dbname_, options_);
|
||||
|
||||
db_ = NULL;
|
||||
@@ -51,17 +51,14 @@ class CorruptionTest {
|
||||
delete tiny_cache_;
|
||||
}
|
||||
|
||||
Status TryReopen(Options* options = NULL) {
|
||||
Status TryReopen() {
|
||||
delete db_;
|
||||
db_ = NULL;
|
||||
Options opt = (options ? *options : options_);
|
||||
opt.env = &env_;
|
||||
opt.block_cache = tiny_cache_;
|
||||
return DB::Open(opt, dbname_, &db_);
|
||||
return DB::Open(options_, dbname_, &db_);
|
||||
}
|
||||
|
||||
void Reopen(Options* options = NULL) {
|
||||
ASSERT_OK(TryReopen(options));
|
||||
void Reopen() {
|
||||
ASSERT_OK(TryReopen());
|
||||
}
|
||||
|
||||
void RepairDB() {
|
||||
@@ -78,7 +75,13 @@ class CorruptionTest {
|
||||
Slice key = Key(i, &key_space);
|
||||
batch.Clear();
|
||||
batch.Put(key, Value(i, &value_space));
|
||||
ASSERT_OK(db_->Write(WriteOptions(), &batch));
|
||||
WriteOptions options;
|
||||
// Corrupt() doesn't work without this sync on windows; stat reports 0 for
|
||||
// the file size.
|
||||
if (i == n - 1) {
|
||||
options.sync = true;
|
||||
}
|
||||
ASSERT_OK(db_->Write(options, &batch));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +96,10 @@ class CorruptionTest {
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
uint64_t key;
|
||||
Slice in(iter->key());
|
||||
if (in == "" || in == "~") {
|
||||
// Ignore boundary keys.
|
||||
continue;
|
||||
}
|
||||
if (!ConsumeDecimalNumber(&in, &key) ||
|
||||
!in.empty() ||
|
||||
key < next_expected) {
|
||||
@@ -116,26 +123,19 @@ class CorruptionTest {
|
||||
ASSERT_GE(max_expected, correct);
|
||||
}
|
||||
|
||||
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt, int level=0) {
|
||||
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
|
||||
// Pick file to corrupt
|
||||
std::vector<std::string> filenames;
|
||||
std::string dirname;
|
||||
if (leveldb::kTableFile!=filetype)
|
||||
dirname=dbname_;
|
||||
else
|
||||
dirname=MakeDirName2(options_, level, "sst");
|
||||
|
||||
ASSERT_OK(env_.GetChildren(dirname, &filenames));
|
||||
|
||||
ASSERT_OK(env_.GetChildren(dbname_, &filenames));
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
std::string fname;
|
||||
int picked_number = -1;
|
||||
for (int i = 0; i < filenames.size(); i++) {
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type) &&
|
||||
type == filetype &&
|
||||
int(number) > picked_number) { // Pick latest file
|
||||
fname = dirname + "/" + filenames[i];
|
||||
fname = dbname_ + "/" + filenames[i];
|
||||
picked_number = number;
|
||||
}
|
||||
}
|
||||
@@ -222,14 +222,12 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
|
||||
const int num = 3 + (Options().write_buffer_size / kValueSize);
|
||||
std::string value_storage;
|
||||
Status s;
|
||||
for (int i = 0;
|
||||
s.ok() && i < num && 0==env_.num_writable_file_errors_;
|
||||
i++) {
|
||||
for (int i = 0; s.ok() && i < num; i++) {
|
||||
WriteBatch batch;
|
||||
batch.Put("a", Value(100, &value_storage));
|
||||
s = db_->Write(WriteOptions(), &batch);
|
||||
}
|
||||
// ASSERT_TRUE(!s.ok()); Background write thread will never report this
|
||||
ASSERT_TRUE(!s.ok());
|
||||
ASSERT_GE(env_.num_writable_file_errors_, 1);
|
||||
env_.writable_file_error_ = false;
|
||||
Reopen();
|
||||
@@ -242,18 +240,34 @@ TEST(CorruptionTest, TableFile) {
|
||||
dbi->TEST_CompactRange(0, NULL, NULL);
|
||||
dbi->TEST_CompactRange(1, NULL, NULL);
|
||||
|
||||
Corrupt(kTableFile, 100, 1, config::kMaxMemCompactLevel);
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
Check(90, 99);
|
||||
}
|
||||
|
||||
TEST(CorruptionTest, TableFileRepair) {
|
||||
options_.block_size = 2 * kValueSize; // Limit scope of corruption
|
||||
options_.paranoid_checks = true;
|
||||
Reopen();
|
||||
Build(100);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
dbi->TEST_CompactRange(0, NULL, NULL);
|
||||
dbi->TEST_CompactRange(1, NULL, NULL);
|
||||
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
RepairDB();
|
||||
Reopen();
|
||||
Check(95, 99);
|
||||
}
|
||||
|
||||
TEST(CorruptionTest, TableFileIndexData) {
|
||||
Build(100000); // Enough to build multiple Tables
|
||||
Build(10000); // Enough to build multiple Tables
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
|
||||
Corrupt(kTableFile, -2000, 500, config::kMaxMemCompactLevel);
|
||||
Corrupt(kTableFile, -2000, 500);
|
||||
Reopen();
|
||||
Check(50000, 99999);
|
||||
Check(5000, 9999);
|
||||
}
|
||||
|
||||
TEST(CorruptionTest, MissingDescriptor) {
|
||||
@@ -305,10 +319,10 @@ TEST(CorruptionTest, CompactionInputError) {
|
||||
Build(10);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
const int last = config::kMaxMemCompactLevel; // Riak does not "move" files
|
||||
const int last = config::kMaxMemCompactLevel;
|
||||
ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
|
||||
|
||||
Corrupt(kTableFile, 100, 1, last);
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
Check(5, 9);
|
||||
|
||||
// Force compactions by writing lots of values
|
||||
@@ -317,50 +331,31 @@ TEST(CorruptionTest, CompactionInputError) {
|
||||
}
|
||||
|
||||
TEST(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
Options options;
|
||||
options.paranoid_checks = true;
|
||||
options.write_buffer_size = 1048576;
|
||||
Reopen(&options);
|
||||
|
||||
int current_corruption=Property("leveldb.ReadBlockError");
|
||||
options_.paranoid_checks = true;
|
||||
options_.write_buffer_size = 512 << 10;
|
||||
Reopen();
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
|
||||
// Fill levels >= 1 so memtable compaction outputs to level 1
|
||||
// matthewv 1/10/14 - what does "levels" have to do with this,
|
||||
// switching to compaction trigger.
|
||||
// 7/10/14 - compaction starts between 4 and 6 files ... assume 4 and 1 move
|
||||
// (will make a new, descriptive constant for 4)
|
||||
for (int level = Property("leveldb.num-files-at-level0")+1;
|
||||
level < config::kL0_GroomingTrigger; level++) {
|
||||
dbi->Put(WriteOptions(), "", "begin");
|
||||
dbi->Put(WriteOptions(), "~", "end");
|
||||
// Make multiple inputs so we need to compact.
|
||||
for (int i = 0; i < 2; i++) {
|
||||
Build(10);
|
||||
dbi->TEST_CompactMemTable();
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
env_.SleepForMicroseconds(100000);
|
||||
}
|
||||
dbi->CompactRange(NULL, NULL);
|
||||
|
||||
Build(10);
|
||||
dbi->TEST_CompactMemTable();
|
||||
ASSERT_TRUE(1 < Property("leveldb.num-files-at-level0"));
|
||||
|
||||
Corrupt(kTableFile, 100, 1, 0);
|
||||
Check(5, 9);
|
||||
|
||||
// Write must eventually fail because of corrupted table
|
||||
Status s;
|
||||
// Write must fail because of corrupted table
|
||||
std::string tmp1, tmp2;
|
||||
for (int i = 0; i < 10000 && s.ok(); i++) {
|
||||
s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
|
||||
}
|
||||
if (s.ok())
|
||||
ASSERT_NE(current_corruption, Property("leveldb.ReadBlockError")) << "no ReadBlockError seen";
|
||||
else
|
||||
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
|
||||
Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
|
||||
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
|
||||
}
|
||||
|
||||
TEST(CorruptionTest, UnrelatedKeys) {
|
||||
Build(10);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
Corrupt(kTableFile, 100, 1, config::kMaxMemCompactLevel);
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
|
||||
std::string tmp1, tmp2;
|
||||
ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
|
||||
|
||||
+61
-46
@@ -33,6 +33,7 @@
|
||||
// readmissing -- read N missing keys in random order
|
||||
// readhot -- read N times in random order from 1% section of DB
|
||||
// seekrandom -- N random seeks
|
||||
// open -- cost of opening a DB
|
||||
// crc32c -- repeated crc32c of 4K of data
|
||||
// acquireload -- load N*1000 times
|
||||
// Meta operations:
|
||||
@@ -83,6 +84,14 @@ static bool FLAGS_histogram = false;
|
||||
// (initialized to default value by "main")
|
||||
static int FLAGS_write_buffer_size = 0;
|
||||
|
||||
// Number of bytes written to each file.
|
||||
// (initialized to default value by "main")
|
||||
static int FLAGS_max_file_size = 0;
|
||||
|
||||
// Approximate size of user data packed per block (before compression.
|
||||
// (initialized to default value by "main")
|
||||
static int FLAGS_block_size = 0;
|
||||
|
||||
// Number of bytes to use as a cache of uncompressed data.
|
||||
// Negative means use default settings.
|
||||
static int FLAGS_cache_size = -1;
|
||||
@@ -94,26 +103,21 @@ static int FLAGS_open_files = 0;
|
||||
// Negative means use default settings.
|
||||
static int FLAGS_bloom_bits = -1;
|
||||
|
||||
// Riak bloom adaptation
|
||||
static int FLAGS_bloom2_bits = -1;
|
||||
|
||||
// Riak param for total memory allocation (flex_cache)
|
||||
static uint64_t FLAGS_leveldb_memory = -1;
|
||||
|
||||
// Riak param for compression setting
|
||||
static int FLAGS_compression = 2;
|
||||
|
||||
// If true, do not destroy the existing database. If you set this
|
||||
// flag and also specify a benchmark that wants a fresh database, that
|
||||
// benchmark will fail.
|
||||
static bool FLAGS_use_existing_db = false;
|
||||
|
||||
// If true, reuse existing log/MANIFEST files when re-opening a database.
|
||||
static bool FLAGS_reuse_logs = false;
|
||||
|
||||
// Use the db with the following name.
|
||||
static const char* FLAGS_db = NULL;
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
leveldb::Env* g_env = NULL;
|
||||
|
||||
// Helper for quickly generating random data.
|
||||
class RandomGenerator {
|
||||
@@ -137,7 +141,7 @@ class RandomGenerator {
|
||||
pos_ = 0;
|
||||
}
|
||||
|
||||
Slice Generate(int len) {
|
||||
Slice Generate(size_t len) {
|
||||
if (pos_ + len > data_.size()) {
|
||||
pos_ = 0;
|
||||
assert(len < data_.size());
|
||||
@@ -147,17 +151,19 @@ class RandomGenerator {
|
||||
}
|
||||
};
|
||||
|
||||
#if defined(__linux)
|
||||
static Slice TrimSpace(Slice s) {
|
||||
int start = 0;
|
||||
size_t start = 0;
|
||||
while (start < s.size() && isspace(s[start])) {
|
||||
start++;
|
||||
}
|
||||
int limit = s.size();
|
||||
size_t limit = s.size();
|
||||
while (limit > start && isspace(s[limit-1])) {
|
||||
limit--;
|
||||
}
|
||||
return Slice(s.data() + start, limit - start);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void AppendWithSpace(std::string* str, Slice msg) {
|
||||
if (msg.empty()) return;
|
||||
@@ -189,7 +195,7 @@ class Stats {
|
||||
done_ = 0;
|
||||
bytes_ = 0;
|
||||
seconds_ = 0;
|
||||
start_ = Env::Default()->NowMicros();
|
||||
start_ = g_env->NowMicros();
|
||||
finish_ = start_;
|
||||
message_.clear();
|
||||
}
|
||||
@@ -207,7 +213,7 @@ class Stats {
|
||||
}
|
||||
|
||||
void Stop() {
|
||||
finish_ = Env::Default()->NowMicros();
|
||||
finish_ = g_env->NowMicros();
|
||||
seconds_ = (finish_ - start_) * 1e-6;
|
||||
}
|
||||
|
||||
@@ -217,7 +223,7 @@ class Stats {
|
||||
|
||||
void FinishedSingleOp() {
|
||||
if (FLAGS_histogram) {
|
||||
double now = Env::Default()->NowMicros();
|
||||
double now = g_env->NowMicros();
|
||||
double micros = now - last_op_finish_;
|
||||
hist_.Add(micros);
|
||||
if (micros > 20000) {
|
||||
@@ -399,7 +405,7 @@ class Benchmark {
|
||||
: cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
|
||||
filter_policy_(FLAGS_bloom_bits >= 0
|
||||
? NewBloomFilterPolicy(FLAGS_bloom_bits)
|
||||
: (FLAGS_bloom2_bits >=0 ? NewBloomFilterPolicy2(FLAGS_bloom2_bits) : NULL)),
|
||||
: NULL),
|
||||
db_(NULL),
|
||||
num_(FLAGS_num),
|
||||
value_size_(FLAGS_value_size),
|
||||
@@ -407,10 +413,10 @@ class Benchmark {
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
heap_counter_(0) {
|
||||
std::vector<std::string> files;
|
||||
Env::Default()->GetChildren(FLAGS_db, &files);
|
||||
for (int i = 0; i < files.size(); i++) {
|
||||
g_env->GetChildren(FLAGS_db, &files);
|
||||
for (size_t i = 0; i < files.size(); i++) {
|
||||
if (Slice(files[i]).starts_with("heap-")) {
|
||||
Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
|
||||
g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
|
||||
}
|
||||
}
|
||||
if (!FLAGS_use_existing_db) {
|
||||
@@ -440,7 +446,7 @@ class Benchmark {
|
||||
benchmarks = sep + 1;
|
||||
}
|
||||
|
||||
// Reset parameters that may be overriddden bwlow
|
||||
// Reset parameters that may be overridden below
|
||||
num_ = FLAGS_num;
|
||||
reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
|
||||
value_size_ = FLAGS_value_size;
|
||||
@@ -451,7 +457,11 @@ class Benchmark {
|
||||
bool fresh_db = false;
|
||||
int num_threads = FLAGS_threads;
|
||||
|
||||
if (name == Slice("fillseq")) {
|
||||
if (name == Slice("open")) {
|
||||
method = &Benchmark::OpenBench;
|
||||
num_ /= 10000;
|
||||
if (num_ < 1) num_ = 1;
|
||||
} else if (name == Slice("fillseq")) {
|
||||
fresh_db = true;
|
||||
method = &Benchmark::WriteSeq;
|
||||
} else if (name == Slice("fillbatch")) {
|
||||
@@ -543,7 +553,6 @@ class Benchmark {
|
||||
SharedState* shared;
|
||||
ThreadState* thread;
|
||||
void (Benchmark::*method)(ThreadState*);
|
||||
pthread_t thread_id;
|
||||
};
|
||||
|
||||
static void ThreadBody(void* v) {
|
||||
@@ -589,8 +598,7 @@ class Benchmark {
|
||||
arg[i].shared = &shared;
|
||||
arg[i].thread = new ThreadState(i);
|
||||
arg[i].thread->shared = &shared;
|
||||
arg[i].thread_id=Env::Default()->StartThread(ThreadBody, &arg[i]);
|
||||
pthread_detach(arg[i].thread_id);
|
||||
g_env->StartThread(ThreadBody, &arg[i]);
|
||||
}
|
||||
|
||||
shared.mu.Lock();
|
||||
@@ -701,12 +709,15 @@ class Benchmark {
|
||||
void Open() {
|
||||
assert(db_ == NULL);
|
||||
Options options;
|
||||
options.env = g_env;
|
||||
options.create_if_missing = !FLAGS_use_existing_db;
|
||||
options.block_cache = cache_;
|
||||
options.write_buffer_size = FLAGS_write_buffer_size;
|
||||
options.max_file_size = FLAGS_max_file_size;
|
||||
options.block_size = FLAGS_block_size;
|
||||
options.max_open_files = FLAGS_open_files;
|
||||
options.filter_policy = filter_policy_;
|
||||
options.compression = (leveldb::CompressionType)FLAGS_compression;
|
||||
options.total_leveldb_mem = FLAGS_leveldb_memory;
|
||||
options.reuse_logs = FLAGS_reuse_logs;
|
||||
Status s = DB::Open(options, FLAGS_db, &db_);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "open error: %s\n", s.ToString().c_str());
|
||||
@@ -714,6 +725,14 @@ class Benchmark {
|
||||
}
|
||||
}
|
||||
|
||||
void OpenBench(ThreadState* thread) {
|
||||
for (int i = 0; i < num_; i++) {
|
||||
delete db_;
|
||||
Open();
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
}
|
||||
|
||||
void WriteSeq(ThreadState* thread) {
|
||||
DoWrite(thread, true);
|
||||
}
|
||||
@@ -823,7 +842,6 @@ class Benchmark {
|
||||
|
||||
void SeekRandom(ThreadState* thread) {
|
||||
ReadOptions options;
|
||||
std::string value;
|
||||
int found = 0;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
Iterator* iter = db_->NewIterator(options);
|
||||
@@ -919,7 +937,7 @@ class Benchmark {
|
||||
char fname[100];
|
||||
snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
|
||||
WritableFile* file;
|
||||
Status s = Env::Default()->NewWritableFile(fname, &file, 2<<20);
|
||||
Status s = g_env->NewWritableFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
return;
|
||||
@@ -928,7 +946,7 @@ class Benchmark {
|
||||
delete file;
|
||||
if (!ok) {
|
||||
fprintf(stderr, "heap profiling not supported\n");
|
||||
Env::Default()->DeleteFile(fname);
|
||||
g_env->DeleteFile(fname);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -937,14 +955,14 @@ class Benchmark {
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
|
||||
FLAGS_max_file_size = leveldb::Options().max_file_size;
|
||||
FLAGS_block_size = leveldb::Options().block_size;
|
||||
FLAGS_open_files = leveldb::Options().max_open_files;
|
||||
FLAGS_leveldb_memory = 25000000000LL;
|
||||
std::string default_db_path;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
double d;
|
||||
int n;
|
||||
uint64_t u;
|
||||
char junk;
|
||||
if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
|
||||
FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
|
||||
@@ -956,6 +974,9 @@ int main(int argc, char** argv) {
|
||||
} else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_use_existing_db = n;
|
||||
} else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_reuse_logs = n;
|
||||
} else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_num = n;
|
||||
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
|
||||
@@ -966,18 +987,16 @@ int main(int argc, char** argv) {
|
||||
FLAGS_value_size = n;
|
||||
} else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_write_buffer_size = n;
|
||||
} else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_max_file_size = n;
|
||||
} else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_block_size = n;
|
||||
} else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_cache_size = n;
|
||||
} else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_bloom_bits = n;
|
||||
} else if (sscanf(argv[i], "--bloom_bits2=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_bloom2_bits = n;
|
||||
} else if (sscanf(argv[i], "--leveldb_memory=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_leveldb_memory = n * 1024 * 1024LL;
|
||||
} else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_open_files = n;
|
||||
} else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_compression = n;
|
||||
} else if (strncmp(argv[i], "--db=", 5) == 0) {
|
||||
FLAGS_db = argv[i] + 5;
|
||||
} else {
|
||||
@@ -986,20 +1005,16 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
}
|
||||
|
||||
leveldb::g_env = leveldb::Env::Default();
|
||||
|
||||
// Choose a location for the test database if none given with --db=<path>
|
||||
if (FLAGS_db == NULL) {
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
leveldb::g_env->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
}
|
||||
|
||||
// benchmark class needs to destruct before Shutdown call
|
||||
{
|
||||
leveldb::Benchmark benchmark;
|
||||
benchmark.Run();
|
||||
}
|
||||
|
||||
leveldb::Env::Shutdown();
|
||||
|
||||
leveldb::Benchmark benchmark;
|
||||
benchmark.Run();
|
||||
return 0;
|
||||
}
|
||||
+455
-1303
File diff suppressed because it is too large.
Load diff
+42
-83
@@ -13,7 +13,7 @@
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "util/cache2.h"
|
||||
#include "port/thread_annotations.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@@ -29,37 +29,26 @@ class DBImpl : public DB {
|
||||
virtual ~DBImpl();
|
||||
|
||||
// Implementations of the DB interface
|
||||
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value, const KeyMetaData * meta=NULL);
|
||||
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
|
||||
virtual Status Delete(const WriteOptions&, const Slice& key);
|
||||
virtual Status Write(const WriteOptions& options, WriteBatch* updates);
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value,
|
||||
KeyMetaData * meta=NULL);
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
Value* value,
|
||||
KeyMetaData * meta=NULL);
|
||||
std::string* value);
|
||||
virtual Iterator* NewIterator(const ReadOptions&);
|
||||
virtual const Snapshot* GetSnapshot();
|
||||
virtual void ReleaseSnapshot(const Snapshot* snapshot);
|
||||
virtual bool GetProperty(const Slice& property, std::string* value);
|
||||
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end);
|
||||
virtual Status VerifyLevels();
|
||||
virtual void CheckAvailableCompactions();
|
||||
virtual Logger* GetLogger() const { return options_.info_log; }
|
||||
|
||||
// Extra methods (for testing) that are not in the public DB interface
|
||||
|
||||
const Options & GetOptions() const { return options_; };
|
||||
|
||||
// Compact any files in the named level that overlap [*begin,*end]
|
||||
void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
|
||||
|
||||
// Force current memtable contents to be compacted, waits for completion
|
||||
Status CompactMemTableSynchronous();
|
||||
Status TEST_CompactMemTable(); // wraps CompactMemTableSynchronous (historical)
|
||||
// Force current memtable contents to be compacted.
|
||||
Status TEST_CompactMemTable();
|
||||
|
||||
// Return an internal iterator over the current state of the database.
|
||||
// The keys of this iterator are internal keys (see format.h).
|
||||
@@ -70,82 +59,64 @@ class DBImpl : public DB {
|
||||
// file at a level >= 1.
|
||||
int64_t TEST_MaxNextLevelOverlappingBytes();
|
||||
|
||||
// These are routines that DBListImpl calls across all open databases
|
||||
void ResizeCaches() {double_cache.ResizeCaches();};
|
||||
size_t GetCacheCapacity() {return(double_cache.GetCapacity(false));}
|
||||
void PurgeExpiredFileCache() {double_cache.PurgeExpiredFiles();};
|
||||
// Record a sample of bytes read at the specified internal key.
|
||||
// Samples are taken approximately once every config::kReadBytesPeriod
|
||||
// bytes.
|
||||
void RecordReadSample(Slice key);
|
||||
|
||||
// in util/hot_backup.cc
|
||||
void HotBackup();
|
||||
bool PurgeWriteBuffer();
|
||||
bool WriteBackupManifest();
|
||||
bool CreateBackupLinks(Version * Version, Options & BackupOptions);
|
||||
bool CopyLOGSegment(long FileEnd);
|
||||
void HotBackupComplete();
|
||||
|
||||
void BackgroundCall2(Compaction * Compact);
|
||||
void BackgroundImmCompactCall();
|
||||
bool IsCompactionScheduled();
|
||||
uint32_t RunningCompactionCount() {mutex_.AssertHeld(); return(running_compactions_);};
|
||||
|
||||
protected:
|
||||
private:
|
||||
friend class DB;
|
||||
struct CompactionState;
|
||||
struct Writer;
|
||||
|
||||
Iterator* NewInternalIterator(const ReadOptions&,
|
||||
SequenceNumber* latest_snapshot);
|
||||
SequenceNumber* latest_snapshot,
|
||||
uint32_t* seed);
|
||||
|
||||
Status NewDB();
|
||||
|
||||
// Recover the descriptor from persistent storage. May do a significant
|
||||
// amount of work to recover recently logged updates. Any changes to
|
||||
// be made to the descriptor are added to *edit.
|
||||
Status Recover(VersionEdit* edit);
|
||||
|
||||
// Riak routine: pause DB::Open if too many compactions
|
||||
// stacked up immediately. Happens in some repairs and
|
||||
// some Riak upgrades
|
||||
void CheckCompactionState();
|
||||
Status Recover(VersionEdit* edit, bool* save_manifest)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
void MaybeIgnoreError(Status* s) const;
|
||||
|
||||
// Delete any unneeded files and stale in-memory entries.
|
||||
void DeleteObsoleteFiles();
|
||||
void KeepOrDelete(const std::string & Filename, int level, const std::set<uint64_t> & Live);
|
||||
|
||||
// Compact the in-memory write buffer to disk. Switches to a new
|
||||
// log-file/memtable and writes a new descriptor iff successful.
|
||||
Status CompactMemTable();
|
||||
// Errors are recorded in bg_error_.
|
||||
void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status RecoverLogFile(uint64_t log_number,
|
||||
VersionEdit* edit,
|
||||
SequenceNumber* max_sequence);
|
||||
Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest,
|
||||
VersionEdit* edit, SequenceNumber* max_sequence)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status WriteLevel0Table(volatile MemTable* mem, VersionEdit* edit, Version* base);
|
||||
|
||||
Status MakeRoomForWrite(bool force /* TRUE forces memtable rotation to disk (for testing) */);
|
||||
Status NewRecoveryLog(uint64_t NewLogNumber);
|
||||
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
Status MakeRoomForWrite(bool force /* compact even if there is room? */)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
WriteBatch* BuildBatchGroup(Writer** last_writer);
|
||||
|
||||
void MaybeScheduleCompaction();
|
||||
void RecordBackgroundError(const Status& s);
|
||||
|
||||
Status BackgroundCompaction(Compaction * Compact=NULL);
|
||||
Status BackgroundExpiry(Compaction * Compact=NULL);
|
||||
void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
static void BGWork(void* db);
|
||||
void BackgroundCall();
|
||||
void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
void CleanupCompaction(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
Status DoCompactionWork(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
void CleanupCompaction(CompactionState* compact);
|
||||
Status DoCompactionWork(CompactionState* compact);
|
||||
int64_t PrioritizeWork(bool IsLevel0);
|
||||
|
||||
Status OpenCompactionOutputFile(CompactionState* compact, size_t sample_value_size);
|
||||
bool Send2PageCache(CompactionState * compact);
|
||||
size_t MaybeRaiseBlockSize(Compaction & CompactionStuff, size_t SampleValueSize);
|
||||
Status OpenCompactionOutputFile(CompactionState* compact);
|
||||
Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
|
||||
Status InstallCompactionResults(CompactionState* compact);
|
||||
|
||||
// initialized before options so its block_cache is available
|
||||
class DoubleCache double_cache;
|
||||
Status InstallCompactionResults(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
// Constant after construction
|
||||
Env* const env_;
|
||||
@@ -159,22 +130,20 @@ class DBImpl : public DB {
|
||||
// table_cache_ provides its own synchronization
|
||||
TableCache* table_cache_;
|
||||
|
||||
|
||||
// Lock over the persistent DB state. Non-NULL iff successfully acquired.
|
||||
FileLock* db_lock_;
|
||||
|
||||
// State below is protected by mutex_
|
||||
port::Mutex mutex_;
|
||||
port::Mutex throttle_mutex_; // used by write throttle to force sequential waits on callers
|
||||
port::AtomicPointer shutting_down_;
|
||||
|
||||
port::CondVar bg_cv_; // Signalled when background work finishes
|
||||
MemTable* mem_;
|
||||
volatile MemTable* imm_; // Memtable being compacted
|
||||
MemTable* imm_; // Memtable being compacted
|
||||
port::AtomicPointer has_imm_; // So bg thread can detect non-NULL imm_
|
||||
WritableFile* logfile_;
|
||||
uint64_t logfile_number_;
|
||||
log::Writer* log_;
|
||||
uint32_t seed_; // For sampling.
|
||||
|
||||
// Queue of writers.
|
||||
std::deque<Writer*> writers_;
|
||||
@@ -186,6 +155,9 @@ class DBImpl : public DB {
|
||||
// part of ongoing compactions.
|
||||
std::set<uint64_t> pending_outputs_;
|
||||
|
||||
// Has a background compaction been scheduled or is running?
|
||||
bool bg_compaction_scheduled_;
|
||||
|
||||
// Information for a manual compaction
|
||||
struct ManualCompaction {
|
||||
int level;
|
||||
@@ -194,7 +166,7 @@ class DBImpl : public DB {
|
||||
const InternalKey* end; // NULL means end of key range
|
||||
InternalKey tmp_storage; // Used to keep track of compaction progress
|
||||
};
|
||||
volatile ManualCompaction* manual_compaction_;
|
||||
ManualCompaction* manual_compaction_;
|
||||
|
||||
VersionSet* versions_;
|
||||
|
||||
@@ -218,18 +190,6 @@ class DBImpl : public DB {
|
||||
};
|
||||
CompactionStats stats_[config::kNumLevels];
|
||||
|
||||
volatile uint64_t throttle_end;
|
||||
volatile uint32_t running_compactions_;
|
||||
volatile size_t current_block_size_; // last dynamic block size computed
|
||||
volatile uint64_t block_size_changed_; // NowMicros() when block size computed
|
||||
volatile uint64_t last_low_mem_; // NowMicros() when low memory last seen
|
||||
|
||||
// accessor to new, dynamic block_cache
|
||||
Cache * block_cache() {return(double_cache.GetBlockCache());};
|
||||
Cache * file_cache() {return(double_cache.GetFileCache());};
|
||||
|
||||
volatile bool hotbackup_pending_;
|
||||
|
||||
// No copying allowed
|
||||
DBImpl(const DBImpl&);
|
||||
void operator=(const DBImpl&);
|
||||
@@ -244,8 +204,7 @@ class DBImpl : public DB {
|
||||
extern Options SanitizeOptions(const std::string& db,
|
||||
const InternalKeyComparator* icmp,
|
||||
const InternalFilterPolicy* ipolicy,
|
||||
const Options& src,
|
||||
Cache * block_cache);
|
||||
const Options& src);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
||||
+34
-58
@@ -5,14 +5,14 @@
|
||||
#include "db/db_iter.h"
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/expiry.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/perf_count.h"
|
||||
#include "port/port.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/random.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@@ -48,20 +48,18 @@ class DBIter: public Iterator {
|
||||
kReverse
|
||||
};
|
||||
|
||||
DBIter(const std::string* dbname, Env* env,
|
||||
const Comparator* cmp, Iterator* iter, SequenceNumber s,
|
||||
const ExpiryModule * expiry)
|
||||
: dbname_(dbname),
|
||||
env_(env),
|
||||
DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
|
||||
uint32_t seed)
|
||||
: db_(db),
|
||||
user_comparator_(cmp),
|
||||
iter_(iter),
|
||||
sequence_(s),
|
||||
direction_(kForward),
|
||||
valid_(false),
|
||||
expiry_(expiry) {
|
||||
rnd_(seed),
|
||||
bytes_counter_(RandomPeriod()) {
|
||||
}
|
||||
virtual ~DBIter() {
|
||||
gPerfCounters->Inc(ePerfIterDelete);
|
||||
delete iter_;
|
||||
}
|
||||
virtual bool Valid() const { return valid_; }
|
||||
@@ -73,26 +71,6 @@ class DBIter: public Iterator {
|
||||
assert(valid_);
|
||||
return (direction_ == kForward) ? iter_->value() : saved_value_;
|
||||
}
|
||||
// Riak specific: if a database iterator, returns key meta data
|
||||
// REQUIRES: Valid() and forward iteration
|
||||
// (reverse iteration is possible, just needs code)
|
||||
virtual KeyMetaData & keymetadata() const
|
||||
{
|
||||
assert(valid_ && kForward==direction_);
|
||||
if (kForward==direction_)
|
||||
{
|
||||
ParsedInternalKey parsed;
|
||||
// this initialization clears a warning. ParsedInternalKey says
|
||||
// it is not initializing for performance reasons ... oh well
|
||||
parsed.type=kTypeValue; parsed.sequence=0; parsed.expiry=0;
|
||||
ParseInternalKey(iter_->key(), &parsed);
|
||||
keymetadata_.m_Type=parsed.type;
|
||||
keymetadata_.m_Sequence=parsed.sequence;
|
||||
keymetadata_.m_Expiry=parsed.expiry;
|
||||
}
|
||||
return(keymetadata_);
|
||||
}
|
||||
|
||||
virtual Status status() const {
|
||||
if (status_.ok()) {
|
||||
return iter_->status();
|
||||
@@ -125,8 +103,12 @@ class DBIter: public Iterator {
|
||||
}
|
||||
}
|
||||
|
||||
const std::string* const dbname_;
|
||||
Env* const env_;
|
||||
// Pick next gap with average value of config::kReadBytesPeriod.
|
||||
ssize_t RandomPeriod() {
|
||||
return rnd_.Uniform(2*config::kReadBytesPeriod);
|
||||
}
|
||||
|
||||
DBImpl* db_;
|
||||
const Comparator* const user_comparator_;
|
||||
Iterator* const iter_;
|
||||
SequenceNumber const sequence_;
|
||||
@@ -136,7 +118,9 @@ class DBIter: public Iterator {
|
||||
std::string saved_value_; // == current raw value when direction_==kReverse
|
||||
Direction direction_;
|
||||
bool valid_;
|
||||
const ExpiryModule * expiry_;
|
||||
|
||||
Random rnd_;
|
||||
ssize_t bytes_counter_;
|
||||
|
||||
// No copying allowed
|
||||
DBIter(const DBIter&);
|
||||
@@ -144,7 +128,14 @@ class DBIter: public Iterator {
|
||||
};
|
||||
|
||||
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
|
||||
if (!ParseInternalKey(iter_->key(), ikey)) {
|
||||
Slice k = iter_->key();
|
||||
ssize_t n = k.size() + iter_->value().size();
|
||||
bytes_counter_ -= n;
|
||||
while (bytes_counter_ < 0) {
|
||||
bytes_counter_ += RandomPeriod();
|
||||
db_->RecordReadSample(k);
|
||||
}
|
||||
if (!ParseInternalKey(k, ikey)) {
|
||||
status_ = Status::Corruption("corrupted internal key in DBIter");
|
||||
return false;
|
||||
} else {
|
||||
@@ -155,7 +146,6 @@ inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
|
||||
void DBIter::Next() {
|
||||
assert(valid_);
|
||||
|
||||
gPerfCounters->Inc(ePerfIterNext);
|
||||
if (direction_ == kReverse) { // Switch directions?
|
||||
direction_ = kForward;
|
||||
// iter_ is pointing just before the entries for this->key(),
|
||||
@@ -171,12 +161,13 @@ void DBIter::Next() {
|
||||
saved_key_.clear();
|
||||
return;
|
||||
}
|
||||
// saved_key_ already contains the key to skip past.
|
||||
} else {
|
||||
// Store in saved_key_ the current key so we skip it below.
|
||||
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
|
||||
}
|
||||
|
||||
// Temporarily use saved_key_ as storage for key to skip.
|
||||
std::string* skip = &saved_key_;
|
||||
SaveKey(ExtractUserKey(iter_->key()), skip);
|
||||
FindNextUserEntry(true, skip);
|
||||
FindNextUserEntry(true, &saved_key_);
|
||||
}
|
||||
|
||||
void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
|
||||
@@ -186,9 +177,6 @@ void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
|
||||
do {
|
||||
ParsedInternalKey ikey;
|
||||
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
|
||||
if (IsExpiryKey(ikey.type) && NULL!=expiry_
|
||||
&& expiry_->KeyRetirementCallback(ikey))
|
||||
ikey.type=kTypeDeletion;
|
||||
switch (ikey.type) {
|
||||
case kTypeDeletion:
|
||||
// Arrange to skip all upcoming entries for this key since
|
||||
@@ -196,9 +184,6 @@ void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
|
||||
SaveKey(ikey.user_key, skip);
|
||||
skipping = true;
|
||||
break;
|
||||
|
||||
case kTypeValueWriteTime:
|
||||
case kTypeValueExplicitExpiry:
|
||||
case kTypeValue:
|
||||
if (skipping &&
|
||||
user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
|
||||
@@ -220,7 +205,6 @@ void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
|
||||
void DBIter::Prev() {
|
||||
assert(valid_);
|
||||
|
||||
gPerfCounters->Inc(ePerfIterPrev);
|
||||
if (direction_ == kForward) { // Switch directions?
|
||||
// iter_ is pointing at the current entry. Scan backwards until
|
||||
// the key changes so we can use the normal reverse scanning code.
|
||||
@@ -258,10 +242,6 @@ void DBIter::FindPrevUserEntry() {
|
||||
// We encountered a non-deleted value in entries for previous keys,
|
||||
break;
|
||||
}
|
||||
if (IsExpiryKey(ikey.type) && NULL!=expiry_
|
||||
&& expiry_->KeyRetirementCallback(ikey))
|
||||
ikey.type=kTypeDeletion;
|
||||
|
||||
value_type = ikey.type;
|
||||
if (value_type == kTypeDeletion) {
|
||||
saved_key_.clear();
|
||||
@@ -292,12 +272,11 @@ void DBIter::FindPrevUserEntry() {
|
||||
}
|
||||
|
||||
void DBIter::Seek(const Slice& target) {
|
||||
gPerfCounters->Inc(ePerfIterSeek);
|
||||
direction_ = kForward;
|
||||
ClearSavedValue();
|
||||
saved_key_.clear();
|
||||
AppendInternalKey(
|
||||
&saved_key_, ParsedInternalKey(target, 0, sequence_, kValueTypeForSeek));
|
||||
&saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
|
||||
iter_->Seek(saved_key_);
|
||||
if (iter_->Valid()) {
|
||||
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
|
||||
@@ -307,7 +286,6 @@ void DBIter::Seek(const Slice& target) {
|
||||
}
|
||||
|
||||
void DBIter::SeekToFirst() {
|
||||
gPerfCounters->Inc(ePerfIterSeekFirst);
|
||||
direction_ = kForward;
|
||||
ClearSavedValue();
|
||||
iter_->SeekToFirst();
|
||||
@@ -319,7 +297,6 @@ void DBIter::SeekToFirst() {
|
||||
}
|
||||
|
||||
void DBIter::SeekToLast() {
|
||||
gPerfCounters->Inc(ePerfIterSeekLast);
|
||||
direction_ = kReverse;
|
||||
ClearSavedValue();
|
||||
iter_->SeekToLast();
|
||||
@@ -329,13 +306,12 @@ void DBIter::SeekToLast() {
|
||||
} // anonymous namespace
|
||||
|
||||
Iterator* NewDBIterator(
|
||||
const std::string* dbname,
|
||||
Env* env,
|
||||
DBImpl* db,
|
||||
const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter,
|
||||
const SequenceNumber& sequence,
|
||||
const ExpiryModule * expiry) {
|
||||
return new DBIter(dbname, env, user_key_comparator, internal_iter, sequence, expiry);
|
||||
SequenceNumber sequence,
|
||||
uint32_t seed) {
|
||||
return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
@@ -7,21 +7,21 @@
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/expiry.h"
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class DBImpl;
|
||||
|
||||
// Return a new iterator that converts internal keys (yielded by
|
||||
// "*internal_iter") that were live at the specified "sequence" number
|
||||
// into appropriate user keys.
|
||||
extern Iterator* NewDBIterator(
|
||||
const std::string* dbname,
|
||||
Env* env,
|
||||
DBImpl* db,
|
||||
const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter,
|
||||
const SequenceNumber& sequence,
|
||||
const ExpiryModule * expiry=NULL);
|
||||
SequenceNumber sequence,
|
||||
uint32_t seed);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
||||
+301
-202
@@ -33,8 +33,11 @@ class AtomicCounter {
|
||||
public:
|
||||
AtomicCounter() : count_(0) { }
|
||||
void Increment() {
|
||||
IncrementBy(1);
|
||||
}
|
||||
void IncrementBy(int count) {
|
||||
MutexLock l(&mu_);
|
||||
count_++;
|
||||
count_ += count;
|
||||
}
|
||||
int Read() {
|
||||
MutexLock l(&mu_);
|
||||
@@ -45,13 +48,20 @@ class AtomicCounter {
|
||||
count_ = 0;
|
||||
}
|
||||
};
|
||||
|
||||
void DelayMilliseconds(int millis) {
|
||||
Env::Default()->SleepForMicroseconds(millis * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
// Special Env used to delay background operations
|
||||
class SpecialEnv : public EnvWrapper {
|
||||
public:
|
||||
// sstable Sync() calls are blocked while this pointer is non-NULL.
|
||||
port::AtomicPointer delay_sstable_sync_;
|
||||
// sstable/log Sync() calls are blocked while this pointer is non-NULL.
|
||||
port::AtomicPointer delay_data_sync_;
|
||||
|
||||
// sstable/log Sync() calls return an error.
|
||||
port::AtomicPointer data_sync_error_;
|
||||
|
||||
// Simulate no-space errors while this pointer is non-NULL.
|
||||
port::AtomicPointer no_space_;
|
||||
@@ -59,30 +69,37 @@ class SpecialEnv : public EnvWrapper {
|
||||
// Simulate non-writable file system while this pointer is non-NULL
|
||||
port::AtomicPointer non_writable_;
|
||||
|
||||
// Force sync of manifest files to fail while this pointer is non-NULL
|
||||
port::AtomicPointer manifest_sync_error_;
|
||||
|
||||
// Force write to manifest files to fail while this pointer is non-NULL
|
||||
port::AtomicPointer manifest_write_error_;
|
||||
|
||||
bool count_random_reads_;
|
||||
AtomicCounter random_read_counter_;
|
||||
|
||||
AtomicCounter sleep_counter_;
|
||||
|
||||
explicit SpecialEnv(Env* base) : EnvWrapper(base) {
|
||||
delay_sstable_sync_.Release_Store(NULL);
|
||||
delay_data_sync_.Release_Store(NULL);
|
||||
data_sync_error_.Release_Store(NULL);
|
||||
no_space_.Release_Store(NULL);
|
||||
non_writable_.Release_Store(NULL);
|
||||
count_random_reads_ = false;
|
||||
manifest_sync_error_.Release_Store(NULL);
|
||||
manifest_write_error_.Release_Store(NULL);
|
||||
}
|
||||
|
||||
Status NewWritableFile(const std::string& f, WritableFile** r, size_t map_size) {
|
||||
class SSTableFile : public WritableFile {
|
||||
Status NewWritableFile(const std::string& f, WritableFile** r) {
|
||||
class DataFile : public WritableFile {
|
||||
private:
|
||||
SpecialEnv* env_;
|
||||
WritableFile* base_;
|
||||
|
||||
public:
|
||||
SSTableFile(SpecialEnv* env, WritableFile* base)
|
||||
DataFile(SpecialEnv* env, WritableFile* base)
|
||||
: env_(env),
|
||||
base_(base) {
|
||||
}
|
||||
~SSTableFile() { delete base_; }
|
||||
~DataFile() { delete base_; }
|
||||
Status Append(const Slice& data) {
|
||||
if (env_->no_space_.Acquire_Load() != NULL) {
|
||||
// Drop writes on the floor
|
||||
@@ -94,21 +111,51 @@ class SpecialEnv : public EnvWrapper {
|
||||
Status Close() { return base_->Close(); }
|
||||
Status Flush() { return base_->Flush(); }
|
||||
Status Sync() {
|
||||
while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
|
||||
env_->SleepForMicroseconds(100000);
|
||||
if (env_->data_sync_error_.Acquire_Load() != NULL) {
|
||||
return Status::IOError("simulated data sync error");
|
||||
}
|
||||
while (env_->delay_data_sync_.Acquire_Load() != NULL) {
|
||||
DelayMilliseconds(100);
|
||||
}
|
||||
return base_->Sync();
|
||||
}
|
||||
};
|
||||
class ManifestFile : public WritableFile {
|
||||
private:
|
||||
SpecialEnv* env_;
|
||||
WritableFile* base_;
|
||||
public:
|
||||
ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
|
||||
~ManifestFile() { delete base_; }
|
||||
Status Append(const Slice& data) {
|
||||
if (env_->manifest_write_error_.Acquire_Load() != NULL) {
|
||||
return Status::IOError("simulated writer error");
|
||||
} else {
|
||||
return base_->Append(data);
|
||||
}
|
||||
}
|
||||
Status Close() { return base_->Close(); }
|
||||
Status Flush() { return base_->Flush(); }
|
||||
Status Sync() {
|
||||
if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
|
||||
return Status::IOError("simulated sync error");
|
||||
} else {
|
||||
return base_->Sync();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (non_writable_.Acquire_Load() != NULL) {
|
||||
return Status::IOError("simulated write error");
|
||||
}
|
||||
|
||||
Status s = target()->NewWritableFile(f, r, 2<<20);
|
||||
Status s = target()->NewWritableFile(f, r);
|
||||
if (s.ok()) {
|
||||
if (strstr(f.c_str(), ".sst") != NULL) {
|
||||
*r = new SSTableFile(this, *r);
|
||||
if (strstr(f.c_str(), ".ldb") != NULL ||
|
||||
strstr(f.c_str(), ".log") != NULL) {
|
||||
*r = new DataFile(this, *r);
|
||||
} else if (strstr(f.c_str(), "MANIFEST") != NULL) {
|
||||
*r = new ManifestFile(this, *r);
|
||||
}
|
||||
}
|
||||
return s;
|
||||
@@ -137,11 +184,6 @@ class SpecialEnv : public EnvWrapper {
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
virtual void SleepForMicroseconds(int micros) {
|
||||
sleep_counter_.Increment();
|
||||
target()->SleepForMicroseconds(micros);
|
||||
}
|
||||
};
|
||||
|
||||
class DBTest {
|
||||
@@ -151,6 +193,7 @@ class DBTest {
|
||||
// Sequence of option configurations to try
|
||||
enum OptionConfig {
|
||||
kDefault,
|
||||
kReuse,
|
||||
kFilter,
|
||||
kUncompressed,
|
||||
kEnd
|
||||
@@ -166,7 +209,7 @@ class DBTest {
|
||||
|
||||
DBTest() : option_config_(kDefault),
|
||||
env_(new SpecialEnv(Env::Default())) {
|
||||
filter_policy_ = NewBloomFilterPolicy2(16);
|
||||
filter_policy_ = NewBloomFilterPolicy(10);
|
||||
dbname_ = test::TmpDir() + "/db_test";
|
||||
DestroyDB(dbname_, Options());
|
||||
db_ = NULL;
|
||||
@@ -195,7 +238,11 @@ class DBTest {
|
||||
// Return the current option configuration.
|
||||
Options CurrentOptions() {
|
||||
Options options;
|
||||
options.reuse_logs = false;
|
||||
switch (option_config_) {
|
||||
case kReuse:
|
||||
options.reuse_logs = true;
|
||||
break;
|
||||
case kFilter:
|
||||
options.filter_policy = filter_policy_;
|
||||
break;
|
||||
@@ -243,23 +290,6 @@ class DBTest {
|
||||
return DB::Open(opts, dbname_, &db_);
|
||||
}
|
||||
|
||||
Status DoubleOpen(Options* options = NULL) {
|
||||
DB * db_fail;
|
||||
delete db_;
|
||||
db_ = NULL;
|
||||
Options opts, opts2;
|
||||
if (options != NULL) {
|
||||
opts = *options;
|
||||
} else {
|
||||
opts = CurrentOptions();
|
||||
opts.create_if_missing = true;
|
||||
}
|
||||
last_options_ = opts;
|
||||
|
||||
DB::Open(opts, dbname_, &db_);
|
||||
return DB::Open(opts2, dbname_, &db_fail);
|
||||
}
|
||||
|
||||
Status Put(const std::string& k, const std::string& v) {
|
||||
return db_->Put(WriteOptions(), k, v);
|
||||
}
|
||||
@@ -281,20 +311,6 @@ class DBTest {
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string GetNoCache(const std::string& k, const Snapshot* snapshot = NULL) {
|
||||
ReadOptions options;
|
||||
options.snapshot = snapshot;
|
||||
options.fill_cache=false;
|
||||
std::string result;
|
||||
Status s = db_->Get(options, k, &result);
|
||||
if (s.IsNotFound()) {
|
||||
result = "NOT_FOUND";
|
||||
} else if (!s.ok()) {
|
||||
result = s.ToString();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Return a string that contains all key,value pairs in order,
|
||||
// formatted like "(k1->v1)(k2->v2)".
|
||||
std::string Contents() {
|
||||
@@ -310,7 +326,7 @@ class DBTest {
|
||||
}
|
||||
|
||||
// Check reverse iteration results are the reverse of forward results
|
||||
int matched = 0;
|
||||
size_t matched = 0;
|
||||
for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
|
||||
ASSERT_LT(matched, forward.size());
|
||||
ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
|
||||
@@ -324,7 +340,7 @@ class DBTest {
|
||||
|
||||
std::string AllEntriesFor(const Slice& user_key) {
|
||||
Iterator* iter = dbfull()->TEST_NewInternalIterator();
|
||||
InternalKey target(user_key, 0, kMaxSequenceNumber, kTypeValue);
|
||||
InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
|
||||
iter->Seek(target.Encode());
|
||||
std::string result;
|
||||
if (!iter->status().ok()) {
|
||||
@@ -345,8 +361,6 @@ class DBTest {
|
||||
}
|
||||
first = false;
|
||||
switch (ikey.type) {
|
||||
case kTypeValueWriteTime:
|
||||
case kTypeValueExplicitExpiry:
|
||||
case kTypeValue:
|
||||
result += iter->value().ToString();
|
||||
break;
|
||||
@@ -460,6 +474,38 @@ class DBTest {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool DeleteAnSSTFile() {
|
||||
std::vector<std::string> filenames;
|
||||
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
|
||||
ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number)));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Returns number of files renamed.
|
||||
int RenameLDBToSST() {
|
||||
std::vector<std::string> filenames;
|
||||
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
int files_renamed = 0;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
|
||||
const std::string from = TableFileName(dbname_, number);
|
||||
const std::string to = SSTTableFileName(dbname_, number);
|
||||
ASSERT_OK(env_->RenameFile(from, to));
|
||||
files_renamed++;
|
||||
}
|
||||
}
|
||||
return files_renamed;
|
||||
}
|
||||
};
|
||||
|
||||
TEST(DBTest, Empty) {
|
||||
@@ -469,11 +515,6 @@ TEST(DBTest, Empty) {
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
TEST(DBTest, DoubleOpen)
|
||||
{
|
||||
ASSERT_NOTOK(DoubleOpen());
|
||||
}
|
||||
|
||||
TEST(DBTest, ReadWrite) {
|
||||
do {
|
||||
ASSERT_OK(Put("foo", "v1"));
|
||||
@@ -506,11 +547,11 @@ TEST(DBTest, GetFromImmutableLayer) {
|
||||
ASSERT_OK(Put("foo", "v1"));
|
||||
ASSERT_EQ("v1", Get("foo"));
|
||||
|
||||
env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
|
||||
env_->delay_data_sync_.Release_Store(env_); // Block sync calls
|
||||
Put("k1", std::string(100000, 'x')); // Fill memtable
|
||||
Put("k2", std::string(100000, 'y')); // Trigger compaction
|
||||
ASSERT_EQ("v1", Get("foo"));
|
||||
env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
|
||||
env_->delay_data_sync_.Release_Store(NULL); // Release sync calls
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
@@ -522,6 +563,17 @@ TEST(DBTest, GetFromVersions) {
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
TEST(DBTest, GetMemUsage) {
|
||||
do {
|
||||
ASSERT_OK(Put("foo", "v1"));
|
||||
std::string val;
|
||||
ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val));
|
||||
int mem_usage = atoi(val.c_str());
|
||||
ASSERT_GT(mem_usage, 0);
|
||||
ASSERT_LT(mem_usage, 5*1024*1024);
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
TEST(DBTest, GetSnapshot) {
|
||||
do {
|
||||
// Try with both a short key and a long key
|
||||
@@ -582,9 +634,6 @@ TEST(DBTest, GetPicksCorrectFile) {
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
#if 0
|
||||
// riak does not execute compaction due to reads
|
||||
|
||||
TEST(DBTest, GetEncountersEmptyLevel) {
|
||||
do {
|
||||
// Arrange for the following to happen:
|
||||
@@ -593,7 +642,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
|
||||
// * sstable B in level 2
|
||||
// Then do enough Get() calls to arrange for an automatic compaction
|
||||
// of sstable A. A bug would cause the compaction to be marked as
|
||||
// occuring at level 1 (instead of the correct level 0).
|
||||
// occurring at level 1 (instead of the correct level 0).
|
||||
|
||||
// Step 1: First place sstables in levels 0 and 2
|
||||
int compaction_count = 0;
|
||||
@@ -618,12 +667,11 @@ TEST(DBTest, GetEncountersEmptyLevel) {
|
||||
}
|
||||
|
||||
// Step 4: Wait for compaction to finish
|
||||
env_->SleepForMicroseconds(1000000);
|
||||
DelayMilliseconds(1000);
|
||||
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(DBTest, IterEmpty) {
|
||||
Iterator* iter = db_->NewIterator(ReadOptions());
|
||||
@@ -948,8 +996,7 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) {
|
||||
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
||||
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
||||
// not riak ASSERT_GT(NumTableFilesAtLevel(1), 1);
|
||||
ASSERT_EQ(NumTableFilesAtLevel(1), 1); // yes riak
|
||||
ASSERT_GT(NumTableFilesAtLevel(1), 1);
|
||||
for (int i = 0; i < 80; i++) {
|
||||
ASSERT_EQ(Get(Key(i)), values[i]);
|
||||
}
|
||||
@@ -963,8 +1010,7 @@ TEST(DBTest, RepeatedWritesToSameKey) {
|
||||
|
||||
// We must have at most one file per level except for level-0,
|
||||
// which may have up to kL0_StopWritesTrigger files.
|
||||
// ... basho adds *2 since level-1 is now overlapped too
|
||||
const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger*2;
|
||||
const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
|
||||
|
||||
Random rnd(301);
|
||||
std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
|
||||
@@ -1008,13 +1054,11 @@ TEST(DBTest, SparseMerge) {
|
||||
|
||||
// Compactions should not cause us to create a situation where
|
||||
// a file overlaps too much data at the next level.
|
||||
// 07/10/14 matthewv - we overlap first two levels. sparse test not appropriate there,
|
||||
// and we set overlaps into 100s of megabytes as "normal"
|
||||
// ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
||||
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
||||
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
||||
// ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
||||
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
||||
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
||||
// ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
||||
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
||||
}
|
||||
|
||||
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
|
||||
@@ -1052,6 +1096,14 @@ TEST(DBTest, ApproximateSizes) {
|
||||
// 0 because GetApproximateSizes() does not account for memtable space
|
||||
ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
|
||||
|
||||
if (options.reuse_logs) {
|
||||
// Recovery will reuse memtable, and GetApproximateSizes() does not
|
||||
// account for memtable usage;
|
||||
Reopen(&options);
|
||||
ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check sizes across recovery by reopening a few times
|
||||
for (int run = 0; run < 3; run++) {
|
||||
Reopen(&options);
|
||||
@@ -1095,6 +1147,11 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
|
||||
ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
|
||||
ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
|
||||
|
||||
if (options.reuse_logs) {
|
||||
// Need to force a memtable compaction since recovery does not do so.
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
||||
}
|
||||
|
||||
// Check sizes across recovery by reopening a few times
|
||||
for (int run = 0; run < 3; run++) {
|
||||
Reopen(&options);
|
||||
@@ -1166,7 +1223,7 @@ TEST(DBTest, Snapshot) {
|
||||
ASSERT_EQ("v4", Get("foo"));
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
#if 0 // trouble under Riak due to assumed file sizes
|
||||
|
||||
TEST(DBTest, HiddenValuesAreRemoved) {
|
||||
do {
|
||||
Random rnd(301);
|
||||
@@ -1197,7 +1254,7 @@ TEST(DBTest, HiddenValuesAreRemoved) {
|
||||
ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(DBTest, DeletionMarkers1) {
|
||||
Put("foo", "v1");
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
||||
@@ -1214,14 +1271,13 @@ TEST(DBTest, DeletionMarkers1) {
|
||||
Delete("foo");
|
||||
Put("foo", "v2");
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // stays at level 0
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); // riak 1.3, DEL merged out by BuildTable
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
|
||||
Slice z("z");
|
||||
dbfull()->TEST_CompactRange(0, NULL, &z);
|
||||
dbfull()->TEST_CompactRange(1, NULL, &z);
|
||||
dbfull()->TEST_CompactRange(last-2, NULL, &z);
|
||||
// DEL eliminated, but v1 remains because we aren't compacting that level
|
||||
// (DEL can be eliminated because v2 hides v1).
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); // Riak 1.4 has merged to level 1
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
|
||||
dbfull()->TEST_CompactRange(last-1, NULL, NULL);
|
||||
// Merging last-1 w/ last, so we are the base level for "foo", so
|
||||
// DEL is removed. (as is v1).
|
||||
@@ -1233,47 +1289,39 @@ TEST(DBTest, DeletionMarkers2) {
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
||||
const int last = config::kMaxMemCompactLevel;
|
||||
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
|
||||
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
||||
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
|
||||
ASSERT_EQ(NumTableFilesAtLevel(last-1), 0);
|
||||
|
||||
// Place a table at level last-1 to prevent merging with preceding mutation
|
||||
Put("a", "begin");
|
||||
Put("z", "end");
|
||||
dbfull()->TEST_CompactMemTable(); // goes to last-1
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
ASSERT_EQ(NumTableFilesAtLevel(last), 1);
|
||||
ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
|
||||
|
||||
Delete("foo");
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level 0
|
||||
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
||||
dbfull()->TEST_CompactRange(0, NULL, NULL); // Riak overlaps level 1
|
||||
dbfull()->TEST_CompactRange(last-2, NULL, NULL);
|
||||
// DEL kept: "last" file overlaps
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
||||
dbfull()->TEST_CompactRange(last-1, NULL, NULL);
|
||||
// Merging last-1 w/ last, so we are the base level for "foo", so
|
||||
// DEL is removed. (as is v1).
|
||||
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
||||
|
||||
dbfull()->TEST_CompactRange(2, NULL, NULL);
|
||||
ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
|
||||
}
|
||||
|
||||
TEST(DBTest, OverlapInLevel0) {
|
||||
do {
|
||||
ASSERT_EQ(config::kMaxMemCompactLevel, 3) << "Fix test to match config";
|
||||
ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
|
||||
|
||||
// Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
|
||||
ASSERT_OK(Put("100", "v100"));
|
||||
ASSERT_OK(Put("999", "v999"));
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
||||
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
||||
ASSERT_OK(Delete("100"));
|
||||
ASSERT_OK(Delete("999"));
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
||||
ASSERT_EQ("0,0,1,1", FilesPerLevel());
|
||||
ASSERT_EQ("0,1,1", FilesPerLevel());
|
||||
|
||||
// Make files spanning the following ranges in level-0:
|
||||
// files[0] 200 .. 900
|
||||
@@ -1286,7 +1334,7 @@ TEST(DBTest, OverlapInLevel0) {
|
||||
ASSERT_OK(Put("600", "v600"));
|
||||
ASSERT_OK(Put("900", "v900"));
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
ASSERT_EQ("2,0,1,1", FilesPerLevel());
|
||||
ASSERT_EQ("2,1,1", FilesPerLevel());
|
||||
|
||||
// Compact away the placeholder files we created initially
|
||||
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
||||
@@ -1316,7 +1364,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_a) {
|
||||
Reopen();
|
||||
Reopen();
|
||||
ASSERT_EQ("(a->v)", Contents());
|
||||
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
||||
DelayMilliseconds(1000); // Wait for compaction to finish
|
||||
ASSERT_EQ("(a->v)", Contents());
|
||||
}
|
||||
|
||||
@@ -1332,7 +1380,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
|
||||
Put("","");
|
||||
Reopen();
|
||||
Put("","");
|
||||
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
||||
DelayMilliseconds(1000); // Wait for compaction to finish
|
||||
Reopen();
|
||||
Put("d","dv");
|
||||
Reopen();
|
||||
@@ -1342,7 +1390,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
|
||||
Delete("b");
|
||||
Reopen();
|
||||
ASSERT_EQ("(->)(c->cv)", Contents());
|
||||
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
||||
DelayMilliseconds(1000); // Wait for compaction to finish
|
||||
ASSERT_EQ("(->)(c->cv)", Contents());
|
||||
}
|
||||
|
||||
@@ -1425,37 +1473,37 @@ TEST(DBTest, CustomComparator) {
|
||||
}
|
||||
|
||||
TEST(DBTest, ManualCompaction) {
|
||||
ASSERT_EQ(config::kMaxMemCompactLevel, 3)
|
||||
ASSERT_EQ(config::kMaxMemCompactLevel, 2)
|
||||
<< "Need to update this test to match kMaxMemCompactLevel";
|
||||
|
||||
MakeTables(3, "p", "q");
|
||||
ASSERT_EQ("1,0,1,1", FilesPerLevel());
|
||||
ASSERT_EQ("1,1,1", FilesPerLevel());
|
||||
|
||||
// Compaction range falls before files
|
||||
Compact("", "c");
|
||||
ASSERT_EQ("0,1,1,1", FilesPerLevel());
|
||||
ASSERT_EQ("1,1,1", FilesPerLevel());
|
||||
|
||||
// Compaction range falls after files
|
||||
Compact("r", "z");
|
||||
ASSERT_EQ("0,1,1,1", FilesPerLevel());
|
||||
ASSERT_EQ("1,1,1", FilesPerLevel());
|
||||
|
||||
// Compaction range overlaps files
|
||||
Compact("p1", "p9");
|
||||
ASSERT_EQ("0,0,0,1", FilesPerLevel());
|
||||
ASSERT_EQ("0,0,1", FilesPerLevel());
|
||||
|
||||
// Populate a different range
|
||||
MakeTables(3, "c", "e");
|
||||
ASSERT_EQ("1,0,1,2", FilesPerLevel());
|
||||
ASSERT_EQ("1,1,2", FilesPerLevel());
|
||||
|
||||
// Compact just the new range
|
||||
Compact("b", "f");
|
||||
ASSERT_EQ("0,0,0,2", FilesPerLevel());
|
||||
ASSERT_EQ("0,0,2", FilesPerLevel());
|
||||
|
||||
// Compact all
|
||||
MakeTables(1, "a", "z");
|
||||
ASSERT_EQ("0,0,1,2", FilesPerLevel());
|
||||
ASSERT_EQ("0,1,2", FilesPerLevel());
|
||||
db_->CompactRange(NULL, NULL);
|
||||
ASSERT_EQ("0,0,0,1", FilesPerLevel());
|
||||
ASSERT_EQ("0,0,1", FilesPerLevel());
|
||||
}
|
||||
|
||||
TEST(DBTest, DBOpen_Options) {
|
||||
@@ -1497,6 +1545,12 @@ TEST(DBTest, DBOpen_Options) {
|
||||
db = NULL;
|
||||
}
|
||||
|
||||
TEST(DBTest, Locking) {
|
||||
DB* db2 = NULL;
|
||||
Status s = DB::Open(CurrentOptions(), dbname_, &db2);
|
||||
ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
|
||||
}
|
||||
|
||||
// Check that number of files does not grow when we are out of space
|
||||
TEST(DBTest, NoSpace) {
|
||||
Options options = CurrentOptions();
|
||||
@@ -1508,19 +1562,15 @@ TEST(DBTest, NoSpace) {
|
||||
Compact("a", "z");
|
||||
const int num_files = CountFiles();
|
||||
env_->no_space_.Release_Store(env_); // Force out-of-space errors
|
||||
env_->sleep_counter_.Reset();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
for (int level = 0; level < config::kNumLevels-1; level++) {
|
||||
dbfull()->TEST_CompactRange(level, NULL, NULL);
|
||||
}
|
||||
}
|
||||
env_->no_space_.Release_Store(NULL);
|
||||
ASSERT_LT(CountFiles(), num_files + 3);
|
||||
|
||||
// Check that compaction attempts slept after errors
|
||||
ASSERT_GE(env_->sleep_counter_.Read(), 5);
|
||||
}
|
||||
#if 0
|
||||
|
||||
TEST(DBTest, NonWritableFileSystem) {
|
||||
Options options = CurrentOptions();
|
||||
options.write_buffer_size = 1000;
|
||||
@@ -1534,13 +1584,119 @@ TEST(DBTest, NonWritableFileSystem) {
|
||||
fprintf(stderr, "iter %d; errors %d\n", i, errors);
|
||||
if (!Put("foo", big).ok()) {
|
||||
errors++;
|
||||
env_->SleepForMicroseconds(100000);
|
||||
DelayMilliseconds(100);
|
||||
}
|
||||
}
|
||||
ASSERT_GT(errors, 0);
|
||||
env_->non_writable_.Release_Store(NULL);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(DBTest, WriteSyncError) {
|
||||
// Check that log sync errors cause the DB to disallow future writes.
|
||||
|
||||
// (a) Cause log sync calls to fail
|
||||
Options options = CurrentOptions();
|
||||
options.env = env_;
|
||||
Reopen(&options);
|
||||
env_->data_sync_error_.Release_Store(env_);
|
||||
|
||||
// (b) Normal write should succeed
|
||||
WriteOptions w;
|
||||
ASSERT_OK(db_->Put(w, "k1", "v1"));
|
||||
ASSERT_EQ("v1", Get("k1"));
|
||||
|
||||
// (c) Do a sync write; should fail
|
||||
w.sync = true;
|
||||
ASSERT_TRUE(!db_->Put(w, "k2", "v2").ok());
|
||||
ASSERT_EQ("v1", Get("k1"));
|
||||
ASSERT_EQ("NOT_FOUND", Get("k2"));
|
||||
|
||||
// (d) make sync behave normally
|
||||
env_->data_sync_error_.Release_Store(NULL);
|
||||
|
||||
// (e) Do a non-sync write; should fail
|
||||
w.sync = false;
|
||||
ASSERT_TRUE(!db_->Put(w, "k3", "v3").ok());
|
||||
ASSERT_EQ("v1", Get("k1"));
|
||||
ASSERT_EQ("NOT_FOUND", Get("k2"));
|
||||
ASSERT_EQ("NOT_FOUND", Get("k3"));
|
||||
}
|
||||
|
||||
TEST(DBTest, ManifestWriteError) {
|
||||
// Test for the following problem:
|
||||
// (a) Compaction produces file F
|
||||
// (b) Log record containing F is written to MANIFEST file, but Sync() fails
|
||||
// (c) GC deletes F
|
||||
// (d) After reopening DB, reads fail since deleted F is named in log record
|
||||
|
||||
// We iterate twice. In the second iteration, everything is the
|
||||
// same except the log record never makes it to the MANIFEST file.
|
||||
for (int iter = 0; iter < 2; iter++) {
|
||||
port::AtomicPointer* error_type = (iter == 0)
|
||||
? &env_->manifest_sync_error_
|
||||
: &env_->manifest_write_error_;
|
||||
|
||||
// Insert foo=>bar mapping
|
||||
Options options = CurrentOptions();
|
||||
options.env = env_;
|
||||
options.create_if_missing = true;
|
||||
options.error_if_exists = false;
|
||||
DestroyAndReopen(&options);
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
|
||||
// Memtable compaction (will succeed)
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
const int last = config::kMaxMemCompactLevel;
|
||||
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
|
||||
|
||||
// Merging compaction (will fail)
|
||||
error_type->Release_Store(env_);
|
||||
dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
|
||||
// Recovery: should not lose data
|
||||
error_type->Release_Store(NULL);
|
||||
Reopen(&options);
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DBTest, MissingSSTFile) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
|
||||
// Dump the memtable to disk.
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
|
||||
Close();
|
||||
ASSERT_TRUE(DeleteAnSSTFile());
|
||||
Options options = CurrentOptions();
|
||||
options.paranoid_checks = true;
|
||||
Status s = TryReopen(&options);
|
||||
ASSERT_TRUE(!s.ok());
|
||||
ASSERT_TRUE(s.ToString().find("issing") != std::string::npos)
|
||||
<< s.ToString();
|
||||
}
|
||||
|
||||
TEST(DBTest, StillReadSST) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
|
||||
// Dump the memtable to disk.
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
Close();
|
||||
ASSERT_GT(RenameLDBToSST(), 0);
|
||||
Options options = CurrentOptions();
|
||||
options.paranoid_checks = true;
|
||||
Status s = TryReopen(&options);
|
||||
ASSERT_TRUE(s.ok());
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
|
||||
TEST(DBTest, FilesDeletedAfterCompaction) {
|
||||
ASSERT_OK(Put("foo", "v2"));
|
||||
Compact("a", "z");
|
||||
@@ -1557,7 +1713,7 @@ TEST(DBTest, BloomFilter) {
|
||||
Options options = CurrentOptions();
|
||||
options.env = env_;
|
||||
options.block_cache = NewLRUCache(0); // Prevent cache hits
|
||||
options.filter_policy = NewBloomFilterPolicy2(16);
|
||||
options.filter_policy = NewBloomFilterPolicy(10);
|
||||
Reopen(&options);
|
||||
|
||||
// Populate multiple layers
|
||||
@@ -1572,12 +1728,12 @@ TEST(DBTest, BloomFilter) {
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
|
||||
// Prevent auto compactions triggered by seeks
|
||||
env_->delay_sstable_sync_.Release_Store(env_);
|
||||
env_->delay_data_sync_.Release_Store(env_);
|
||||
|
||||
// Lookup present keys. Should rarely read from small sstable.
|
||||
env_->random_read_counter_.Reset();
|
||||
for (int i = 0; i < N; i++) {
|
||||
ASSERT_EQ(Key(i), GetNoCache(Key(i)));
|
||||
ASSERT_EQ(Key(i), Get(Key(i)));
|
||||
}
|
||||
int reads = env_->random_read_counter_.Read();
|
||||
fprintf(stderr, "%d present => %d reads\n", N, reads);
|
||||
@@ -1587,13 +1743,13 @@ TEST(DBTest, BloomFilter) {
|
||||
// Lookup present keys. Should rarely read from either sstable.
|
||||
env_->random_read_counter_.Reset();
|
||||
for (int i = 0; i < N; i++) {
|
||||
ASSERT_EQ("NOT_FOUND", GetNoCache(Key(i) + ".missing"));
|
||||
ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing"));
|
||||
}
|
||||
reads = env_->random_read_counter_.Read();
|
||||
fprintf(stderr, "%d missing => %d reads\n", N, reads);
|
||||
ASSERT_LE(reads, 3*N/100);
|
||||
|
||||
env_->delay_sstable_sync_.Release_Store(NULL);
|
||||
env_->delay_data_sync_.Release_Store(NULL);
|
||||
Close();
|
||||
delete options.block_cache;
|
||||
delete options.filter_policy;
|
||||
@@ -1653,7 +1809,7 @@ static void MTThreadBody(void* arg) {
|
||||
ASSERT_EQ(k, key);
|
||||
ASSERT_GE(w, 0);
|
||||
ASSERT_LT(w, kNumThreads);
|
||||
ASSERT_LE(c, reinterpret_cast<uintptr_t>(
|
||||
ASSERT_LE(static_cast<uintptr_t>(c), reinterpret_cast<uintptr_t>(
|
||||
t->state->counter[w].Acquire_Load()));
|
||||
}
|
||||
}
|
||||
@@ -1678,35 +1834,27 @@ TEST(DBTest, MultiThreaded) {
|
||||
|
||||
// Start threads
|
||||
MTThread thread[kNumThreads];
|
||||
pthread_t tid;
|
||||
for (int id = 0; id < kNumThreads; id++) {
|
||||
thread[id].state = &mt;
|
||||
thread[id].id = id;
|
||||
tid=env_->StartThread(MTThreadBody, &thread[id]);
|
||||
pthread_detach(tid);
|
||||
env_->StartThread(MTThreadBody, &thread[id]);
|
||||
}
|
||||
|
||||
// Let them run for a while
|
||||
env_->SleepForMicroseconds(kTestSeconds * 1000000);
|
||||
DelayMilliseconds(kTestSeconds * 1000);
|
||||
|
||||
// Stop the threads and wait for them to finish
|
||||
mt.stop.Release_Store(&mt);
|
||||
for (int id = 0; id < kNumThreads; id++) {
|
||||
while (mt.thread_done[id].Acquire_Load() == NULL) {
|
||||
env_->SleepForMicroseconds(100000);
|
||||
DelayMilliseconds(100);
|
||||
}
|
||||
}
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
namespace {
|
||||
struct KVEntry
|
||||
{
|
||||
std::string m_Value;
|
||||
KeyMetaData m_Meta;
|
||||
};
|
||||
|
||||
typedef std::map<std::string, KVEntry> KVMap;
|
||||
typedef std::map<std::string, std::string> KVMap;
|
||||
}
|
||||
|
||||
class ModelDB: public DB {
|
||||
@@ -1718,21 +1866,14 @@ class ModelDB: public DB {
|
||||
|
||||
explicit ModelDB(const Options& options): options_(options) { }
|
||||
~ModelDB() { }
|
||||
virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v, const KeyMetaData * meta=NULL) {
|
||||
return DB::Put(o, k, v, meta);
|
||||
virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
|
||||
return DB::Put(o, k, v);
|
||||
}
|
||||
virtual Status Delete(const WriteOptions& o, const Slice& key) {
|
||||
return DB::Delete(o, key);
|
||||
}
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key, std::string* value,
|
||||
KeyMetaData * meta = NULL) {
|
||||
assert(false); // Not implemented
|
||||
return Status::NotFound(key);
|
||||
}
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key, Value* value,
|
||||
KeyMetaData * meta = NULL) {
|
||||
const Slice& key, std::string* value) {
|
||||
assert(false); // Not implemented
|
||||
return Status::NotFound(key);
|
||||
}
|
||||
@@ -1760,13 +1901,8 @@ class ModelDB: public DB {
|
||||
class Handler : public WriteBatch::Handler {
|
||||
public:
|
||||
KVMap* map_;
|
||||
virtual void Put(const Slice& key, const Slice& value,
|
||||
const ValueType & type, const ExpiryTimeMicros & expiry) {
|
||||
KVEntry ent;
|
||||
ent.m_Value=value.ToString();
|
||||
ent.m_Meta.m_Type=type;
|
||||
ent.m_Meta.m_Expiry=expiry;
|
||||
(*map_)[key.ToString()] = ent;
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
(*map_)[key.ToString()] = value.ToString();
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
map_->erase(key.ToString());
|
||||
@@ -1812,7 +1948,7 @@ class ModelDB: public DB {
|
||||
virtual void Next() { ++iter_; }
|
||||
virtual void Prev() { --iter_; }
|
||||
virtual Slice key() const { return iter_->first; }
|
||||
virtual Slice value() const { return iter_->second.m_Value; }
|
||||
virtual Slice value() const { return iter_->second; }
|
||||
virtual Status status() const { return Status::OK(); }
|
||||
private:
|
||||
const KVMap* const map_;
|
||||
@@ -1949,44 +2085,6 @@ TEST(DBTest, Randomized) {
|
||||
} while (ChangeOptions());
|
||||
}
|
||||
|
||||
|
||||
class SimpleBugs
|
||||
{
|
||||
// need a class for the test harness
|
||||
};
|
||||
|
||||
|
||||
TEST(SimpleBugs, TieredRecoveryLog)
|
||||
{
|
||||
// DB::Open created first recovery log directly
|
||||
// which lead to it NOT being in tiered storage location.
|
||||
// nope std::string dbname = test::TmpDir() + "/leveldb_nontiered";
|
||||
std::string dbname = "leveldb";
|
||||
std::string fastname = test::TmpDir() + "/leveldb_fast";
|
||||
std::string slowname = test::TmpDir() + "/leveldb_slow";
|
||||
std::string combined;
|
||||
|
||||
DB* db = NULL;
|
||||
Options opts;
|
||||
|
||||
opts.tiered_slow_level = 4;
|
||||
opts.tiered_fast_prefix = fastname;
|
||||
opts.tiered_slow_prefix = slowname;
|
||||
opts.create_if_missing = true;
|
||||
|
||||
Env::Default()->CreateDir(fastname);
|
||||
Env::Default()->CreateDir(slowname);
|
||||
|
||||
Status s = DB::Open(opts, dbname, &db);
|
||||
ASSERT_OK(s);
|
||||
ASSERT_TRUE(db != NULL);
|
||||
|
||||
delete db;
|
||||
DestroyDB(dbname, opts);
|
||||
|
||||
} // TieredRecoveryLog
|
||||
|
||||
|
||||
std::string MakeKey(unsigned int num) {
|
||||
char buf[30];
|
||||
snprintf(buf, sizeof(buf), "%016u", num);
|
||||
@@ -2015,13 +2113,14 @@ void BM_LogAndApply(int iters, int num_base_files) {
|
||||
InternalKeyComparator cmp(BytewiseComparator());
|
||||
Options options;
|
||||
VersionSet vset(dbname, &options, NULL, &cmp);
|
||||
ASSERT_OK(vset.Recover());
|
||||
bool save_manifest;
|
||||
ASSERT_OK(vset.Recover(&save_manifest));
|
||||
VersionEdit vbase;
|
||||
uint64_t fnum = 1;
|
||||
for (int i = 0; i < num_base_files; i++) {
|
||||
InternalKey start(MakeKey(2*fnum), 0, 1, kTypeValue);
|
||||
InternalKey limit(MakeKey(2*fnum+1), 0, 1, kTypeDeletion);
|
||||
vbase.AddFile2(2, fnum++, 1 /* file size */, start, limit, 0,0,0);
|
||||
InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
|
||||
InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
|
||||
vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
|
||||
}
|
||||
ASSERT_OK(vset.LogAndApply(&vbase, &mu));
|
||||
|
||||
@@ -2030,9 +2129,9 @@ void BM_LogAndApply(int iters, int num_base_files) {
|
||||
for (int i = 0; i < iters; i++) {
|
||||
VersionEdit vedit;
|
||||
vedit.DeleteFile(2, fnum);
|
||||
InternalKey start(MakeKey(2*fnum), 0, 1, kTypeValue);
|
||||
InternalKey limit(MakeKey(2*fnum+1), 0, 1, kTypeDeletion);
|
||||
vedit.AddFile2(2, fnum++, 1 /* file size */, start, limit, 0,0,0);
|
||||
InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
|
||||
InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
|
||||
vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
|
||||
vset.LogAndApply(&vedit, &mu);
|
||||
}
|
||||
uint64_t stop_micros = env->NowMicros();
|
||||
|
||||
+8
-158
@@ -3,9 +3,7 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
//#include "leveldb/expiry.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/version_set.h"
|
||||
#include "port/port.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
@@ -13,66 +11,26 @@ namespace leveldb {
|
||||
|
||||
static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
|
||||
assert(seq <= kMaxSequenceNumber);
|
||||
// assert(t <= kValueTypeForSeek); requires revisit once expiry live
|
||||
assert(t <= kTypeValueExplicitExpiry); // temp replacement for above
|
||||
assert(t <= kValueTypeForSeek);
|
||||
return (seq << 8) | t;
|
||||
}
|
||||
|
||||
void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
|
||||
result->append(key.user_key.data(), key.user_key.size());
|
||||
if (IsExpiryKey(key.type))
|
||||
PutFixed64(result, key.expiry);
|
||||
PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
|
||||
}
|
||||
|
||||
std::string ParsedInternalKey::DebugString() const {
|
||||
char buf[50];
|
||||
if (IsExpiryKey(type))
|
||||
snprintf(buf, sizeof(buf), "' @ %llu %llu : %d",
|
||||
(unsigned long long) expiry,
|
||||
(unsigned long long) sequence,
|
||||
int(type));
|
||||
else
|
||||
snprintf(buf, sizeof(buf), "' @ %llu : %d",
|
||||
(unsigned long long) sequence,
|
||||
int(type));
|
||||
snprintf(buf, sizeof(buf), "' @ %llu : %d",
|
||||
(unsigned long long) sequence,
|
||||
int(type));
|
||||
std::string result = "'";
|
||||
result += HexString(user_key.ToString());
|
||||
result += EscapeString(user_key.ToString());
|
||||
result += buf;
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string ParsedInternalKey::DebugStringHex() const {
|
||||
char buf[50];
|
||||
if (IsExpiryKey(type))
|
||||
snprintf(buf, sizeof(buf), "' @ %llu %llu : %d",
|
||||
(unsigned long long) expiry,
|
||||
(unsigned long long) sequence,
|
||||
int(type));
|
||||
else
|
||||
snprintf(buf, sizeof(buf), "' @ %llu : %d",
|
||||
(unsigned long long) sequence,
|
||||
int(type));
|
||||
std::string result = "'";
|
||||
result += HexString(user_key);
|
||||
result += buf;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
const char * KeyTypeString(ValueType val_type) {
|
||||
const char * ret_ptr;
|
||||
switch(val_type)
|
||||
{
|
||||
case kTypeDeletion: ret_ptr="kTypeDelete"; break;
|
||||
case kTypeValue: ret_ptr="kTypeValue"; break;
|
||||
case kTypeValueWriteTime: ret_ptr="kTypeValueWriteTime"; break;
|
||||
case kTypeValueExplicitExpiry: ret_ptr="kTypeValueExplicitExpiry"; break;
|
||||
default: ret_ptr="(unknown ValueType)"; break;
|
||||
} // switch
|
||||
return(ret_ptr);
|
||||
}
|
||||
|
||||
std::string InternalKey::DebugString() const {
|
||||
std::string result;
|
||||
ParsedInternalKey parsed;
|
||||
@@ -96,10 +54,8 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
|
||||
// decreasing type (though sequence# should be enough to disambiguate)
|
||||
int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
|
||||
if (r == 0) {
|
||||
uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
|
||||
uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
|
||||
if (IsExpiryKey((ValueType)*(unsigned char *)&anum)) *(unsigned char*)&anum=(unsigned char)kTypeValue;
|
||||
if (IsExpiryKey((ValueType)*(unsigned char *)&bnum)) *(unsigned char*)&bnum=(unsigned char)kTypeValue;
|
||||
const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
|
||||
const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
|
||||
if (anum > bnum) {
|
||||
r = -1;
|
||||
} else if (anum < bnum) {
|
||||
@@ -162,8 +118,7 @@ bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const {
|
||||
return user_policy_->KeyMayMatch(ExtractUserKey(key), f);
|
||||
}
|
||||
|
||||
LookupKey::LookupKey(const Slice& user_key, SequenceNumber s, KeyMetaData * meta) {
|
||||
meta_=meta;
|
||||
LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
|
||||
size_t usize = user_key.size();
|
||||
size_t needed = usize + 13; // A conservative estimate
|
||||
char* dst;
|
||||
@@ -182,109 +137,4 @@ bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const {
|
||||
end_ = dst;
|
||||
}
|
||||
|
||||
|
||||
KeyRetirement::KeyRetirement(
|
||||
const Comparator * Comparator,
|
||||
SequenceNumber SmallestSnapshot,
|
||||
const Options * Opts,
|
||||
Compaction * const Compaction)
|
||||
: has_current_user_key(false), last_sequence_for_key(kMaxSequenceNumber),
|
||||
user_comparator(Comparator), smallest_snapshot(SmallestSnapshot),
|
||||
options(Opts), compaction(Compaction),
|
||||
valid(false), dropped(0), expired(0)
|
||||
{
|
||||
// NULL is ok for compaction
|
||||
valid=(NULL!=user_comparator);
|
||||
|
||||
return;
|
||||
} // KeyRetirement::KeyRetirement
|
||||
|
||||
|
||||
KeyRetirement::~KeyRetirement()
|
||||
{
|
||||
if (0!=expired)
|
||||
gPerfCounters->Add(ePerfExpiredKeys, expired);
|
||||
} // KeyRetirement::~KeyRetirement
|
||||
|
||||
|
||||
bool
|
||||
KeyRetirement::operator()(
|
||||
Slice & key)
|
||||
{
|
||||
ParsedInternalKey ikey;
|
||||
bool drop = false, expire_flag;
|
||||
|
||||
if (valid)
|
||||
{
|
||||
if (!ParseInternalKey(key, &ikey))
|
||||
{
|
||||
// Do not hide error keys
|
||||
current_user_key.clear();
|
||||
has_current_user_key = false;
|
||||
last_sequence_for_key = kMaxSequenceNumber;
|
||||
} // else
|
||||
else
|
||||
{
|
||||
if (!has_current_user_key ||
|
||||
user_comparator->Compare(ikey.user_key,
|
||||
Slice(current_user_key)) != 0)
|
||||
{
|
||||
// First occurrence of this user key
|
||||
current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
|
||||
has_current_user_key = true;
|
||||
last_sequence_for_key = kMaxSequenceNumber;
|
||||
} // if
|
||||
|
||||
if (last_sequence_for_key <= smallest_snapshot)
|
||||
{
|
||||
// Hidden by an newer entry for same user key
|
||||
drop = true; // (A)
|
||||
} // if
|
||||
|
||||
else
|
||||
{
|
||||
expire_flag=false;
|
||||
if (NULL!=options && options->ExpiryActivated())
|
||||
expire_flag=options->expiry_module->KeyRetirementCallback(ikey);
|
||||
|
||||
if ((ikey.type == kTypeDeletion || expire_flag)
|
||||
&& ikey.sequence <= smallest_snapshot
|
||||
&& NULL!=compaction // mem to level0 ignores this test
|
||||
&& compaction->IsBaseLevelForKey(ikey.user_key))
|
||||
{
|
||||
// For this user key:
|
||||
// (1) there is no data in higher levels
|
||||
// (2) data in lower levels will have larger sequence numbers
|
||||
// (3) data in layers that are being compacted here and have
|
||||
// smaller sequence numbers will be dropped in the next
|
||||
// few iterations of this loop (by rule (A) above).
|
||||
// Therefore this deletion marker is obsolete and can be dropped.
|
||||
drop = true;
|
||||
|
||||
if (expire_flag)
|
||||
++expired;
|
||||
else
|
||||
++dropped;
|
||||
} // if
|
||||
} // else
|
||||
|
||||
last_sequence_for_key = ikey.sequence;
|
||||
} // else
|
||||
} // if
|
||||
|
||||
#if 0
|
||||
// needs clean up to be used again
|
||||
Log(options_.info_log,
|
||||
" Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, "
|
||||
"%d smallest_snapshot: %d",
|
||||
ikey.user_key.ToString().c_str(),
|
||||
(int)ikey.sequence, ikey.type, kTypeValue, drop,
|
||||
compact->compaction->IsBaseLevelForKey(ikey.user_key),
|
||||
(int)last_sequence_for_key, (int)compact->smallest_snapshot);
|
||||
#endif
|
||||
return(drop);
|
||||
|
||||
} // KeyRetirement::operator(Slice & )
|
||||
|
||||
|
||||
} // namespace leveldb
|
||||
+38
-182
@@ -2,14 +2,13 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
|
||||
#include <stdio.h>
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/table_builder.h"
|
||||
#include "util/coding.h"
|
||||
@@ -17,33 +16,19 @@
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Compaction;
|
||||
|
||||
// Grouping of constants. We may want to make some of these
|
||||
// parameters set via options.
|
||||
namespace config {
|
||||
static const int kNumLevels = 7;
|
||||
static const int kNumOverlapLevels = 2;
|
||||
|
||||
// Level-0 compaction is started when we hit this many files.
|
||||
// Google: static const size_t kL0_CompactionTrigger = 4;
|
||||
static const size_t kL0_CompactionTrigger = 6;
|
||||
|
||||
// Level-0 (any overlapped level) number of files where a grooming
|
||||
// compaction could start
|
||||
static const size_t kL0_GroomingTrigger = 4;
|
||||
static const size_t kL0_GroomingTrigger10min = 2;
|
||||
static const size_t kL0_GroomingTrigger20min = 1;
|
||||
|
||||
// ... time limits in microseconds
|
||||
static const size_t kL0_Grooming10minMicros = 10 * 60 * 1000000;
|
||||
static const size_t kL0_Grooming20minMicros = 20 * 60 * 1000000;
|
||||
static const int kL0_CompactionTrigger = 4;
|
||||
|
||||
// Soft limit on number of level-0 files. We slow down writes at this point.
|
||||
static const size_t kL0_SlowdownWritesTrigger = 8;
|
||||
static const int kL0_SlowdownWritesTrigger = 8;
|
||||
|
||||
// Maximum number of level-0 files. We stop writes at this point.
|
||||
static const size_t kL0_StopWritesTrigger = 12;
|
||||
static const int kL0_StopWritesTrigger = 12;
|
||||
|
||||
// Maximum level to which a new compacted memtable is pushed if it
|
||||
// does not create overlap. We try to push to level 2 to avoid the
|
||||
@@ -51,28 +36,31 @@ static const size_t kL0_StopWritesTrigger = 12;
|
||||
// expensive manifest file operations. We do not push all the way to
|
||||
// the largest level since that can generate a lot of wasted disk
|
||||
// space if the same key space is being repeatedly overwritten.
|
||||
// Basho: push to kNumOverlapLevels +1 ... beyond "landing level"
|
||||
static const unsigned kMaxMemCompactLevel = kNumOverlapLevels+1;
|
||||
static const int kMaxMemCompactLevel = 2;
|
||||
|
||||
// Approximate gap in bytes between samples of data read during iteration.
|
||||
static const int kReadBytesPeriod = 1048576;
|
||||
|
||||
} // namespace config
|
||||
|
||||
class InternalKey;
|
||||
|
||||
// Value types encoded as the last component of internal keys.
|
||||
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
|
||||
// data structures.
|
||||
enum ValueType {
|
||||
kTypeDeletion = 0x0,
|
||||
kTypeValue = 0x1
|
||||
};
|
||||
// kValueTypeForSeek defines the ValueType that should be passed when
|
||||
// constructing a ParsedInternalKey object for seeking to a particular
|
||||
// sequence number (since we sort sequence numbers in decreasing order
|
||||
// and the value type is embedded as the low 8 bits in the sequence
|
||||
// number in internal keys, we need to use the highest-numbered
|
||||
// ValueType, not the lowest).
|
||||
// Riak note: kValueTypeForSeek is placed within temporary keys
|
||||
// for comparisons. Using kTypeValueExplicitExpiry would
|
||||
// force more code changes to increase internal key size.
|
||||
// But ValueTypeForSeek is redundant to sequence number for
|
||||
// disambiguaty. Therefore going for easiest path and NOT changing.
|
||||
static const ValueType kValueTypeForSeek = kTypeValue;
|
||||
|
||||
typedef uint64_t SequenceNumber;
|
||||
typedef uint64_t ExpiryTimeMicros;
|
||||
|
||||
// We leave eight bits empty at the bottom so a type and sequence#
|
||||
// can be packed together into 64-bits.
|
||||
@@ -81,17 +69,20 @@ static const SequenceNumber kMaxSequenceNumber =
|
||||
|
||||
struct ParsedInternalKey {
|
||||
Slice user_key;
|
||||
ExpiryTimeMicros expiry;
|
||||
SequenceNumber sequence;
|
||||
ValueType type;
|
||||
|
||||
ParsedInternalKey() { } // Intentionally left uninitialized (for speed)
|
||||
ParsedInternalKey(const Slice& u, const ExpiryTimeMicros & exp, const SequenceNumber& seq, ValueType t)
|
||||
: user_key(u), expiry(exp), sequence(seq), type(t) { }
|
||||
ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
|
||||
: user_key(u), sequence(seq), type(t) { }
|
||||
std::string DebugString() const;
|
||||
std::string DebugStringHex() const;
|
||||
};
|
||||
|
||||
// Return the length of the encoding of "key".
|
||||
inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
|
||||
return key.user_key.size() + 8;
|
||||
}
|
||||
|
||||
// Append the serialization of "key" to *result.
|
||||
extern void AppendInternalKey(std::string* result,
|
||||
const ParsedInternalKey& key);
|
||||
@@ -103,74 +94,18 @@ extern void AppendInternalKey(std::string* result,
|
||||
extern bool ParseInternalKey(const Slice& internal_key,
|
||||
ParsedInternalKey* result);
|
||||
|
||||
inline ValueType ExtractValueType(const Slice& internal_key) {
|
||||
assert(internal_key.size() >= 8);
|
||||
const size_t n = internal_key.size();
|
||||
unsigned char c = DecodeLeastFixed64(internal_key.data() + n - sizeof(SequenceNumber));
|
||||
return static_cast<ValueType>(c);
|
||||
}
|
||||
|
||||
inline size_t KeySuffixSize(ValueType val_type) {
|
||||
size_t ret_val;
|
||||
switch(val_type)
|
||||
{
|
||||
case kTypeDeletion:
|
||||
case kTypeValue:
|
||||
ret_val=sizeof(SequenceNumber);
|
||||
break;
|
||||
|
||||
case kTypeValueWriteTime:
|
||||
case kTypeValueExplicitExpiry:
|
||||
ret_val=sizeof(SequenceNumber) + sizeof(ExpiryTimeMicros);
|
||||
break;
|
||||
|
||||
default:
|
||||
// assert(0); cannot use because bloom filter block's name is passed as internal key
|
||||
ret_val=sizeof(SequenceNumber);
|
||||
break;
|
||||
} // switch
|
||||
return(ret_val);
|
||||
}
|
||||
|
||||
const char * KeyTypeString(ValueType val_type);
|
||||
|
||||
inline size_t KeySuffixSize(const Slice & internal_key) {
|
||||
return(KeySuffixSize(ExtractValueType(internal_key)));
|
||||
}
|
||||
|
||||
// Returns the user key portion of an internal key.
|
||||
inline Slice ExtractUserKey(const Slice& internal_key) {
|
||||
assert(internal_key.size() >= 8);
|
||||
return Slice(internal_key.data(), internal_key.size() - KeySuffixSize(internal_key));
|
||||
return Slice(internal_key.data(), internal_key.size() - 8);
|
||||
}
|
||||
|
||||
// Returns the sequence number with ValueType removed
|
||||
inline SequenceNumber ExtractSequenceNumber(const Slice& internal_key) {
|
||||
inline ValueType ExtractValueType(const Slice& internal_key) {
|
||||
assert(internal_key.size() >= 8);
|
||||
return(DecodeFixed64(internal_key.data() + internal_key.size() - 8)>>8);
|
||||
}
|
||||
|
||||
// Return the length of the encoding of "key".
|
||||
inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
|
||||
return key.user_key.size() + KeySuffixSize(key.type);
|
||||
}
|
||||
|
||||
// Riak: is this an expiry key and therefore contain extra ExpiryTime field
|
||||
inline bool IsExpiryKey(ValueType val_type) {
|
||||
return(kTypeValueWriteTime==val_type || kTypeValueExplicitExpiry==val_type);
|
||||
}
|
||||
|
||||
// Riak: is this an expiry key and therefore contain extra ExpiryTime field
|
||||
inline bool IsExpiryKey(const Slice & internal_key) {
|
||||
return(internal_key.size()>=KeySuffixSize(kTypeValueWriteTime)
|
||||
&& IsExpiryKey(ExtractValueType(internal_key)));
|
||||
}
|
||||
|
||||
// Riak: extracts expiry value
|
||||
inline ExpiryTimeMicros ExtractExpiry(const Slice& internal_key) {
|
||||
assert(internal_key.size() >= KeySuffixSize(kTypeValueWriteTime));
|
||||
assert(IsExpiryKey(internal_key));
|
||||
return(DecodeFixed64(internal_key.data() + internal_key.size() - KeySuffixSize(kTypeValueWriteTime)));
|
||||
const size_t n = internal_key.size();
|
||||
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
||||
unsigned char c = num & 0xff;
|
||||
return static_cast<ValueType>(c);
|
||||
}
|
||||
|
||||
// A comparator for internal keys that uses a specified comparator for
|
||||
@@ -194,7 +129,7 @@ class InternalKeyComparator : public Comparator {
|
||||
|
||||
// Filter policy wrapper that converts from internal keys to user keys
|
||||
class InternalFilterPolicy : public FilterPolicy {
|
||||
protected:
|
||||
private:
|
||||
const FilterPolicy* const user_policy_;
|
||||
public:
|
||||
explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { }
|
||||
@@ -203,12 +138,6 @@ class InternalFilterPolicy : public FilterPolicy {
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
|
||||
};
|
||||
|
||||
class InternalFilterPolicy2 : public InternalFilterPolicy {
|
||||
public:
|
||||
explicit InternalFilterPolicy2(const FilterPolicy* p) : InternalFilterPolicy(p) { }
|
||||
virtual ~InternalFilterPolicy2() {delete user_policy_;};
|
||||
};
|
||||
|
||||
// Modules in this directory should keep internal keys wrapped inside
|
||||
// the following class instead of plain strings so that we do not
|
||||
// incorrectly use string comparisons instead of an InternalKeyComparator.
|
||||
@@ -217,8 +146,8 @@ class InternalKey {
|
||||
std::string rep_;
|
||||
public:
|
||||
InternalKey() { } // Leave rep_ as empty to indicate it is invalid
|
||||
InternalKey(const Slice& user_key, ExpiryTimeMicros exp, SequenceNumber s, ValueType t) {
|
||||
AppendInternalKey(&rep_, ParsedInternalKey(user_key, exp, s, t));
|
||||
InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
|
||||
AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
|
||||
}
|
||||
|
||||
void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
|
||||
@@ -228,7 +157,6 @@ class InternalKey {
|
||||
}
|
||||
|
||||
Slice user_key() const { return ExtractUserKey(rep_); }
|
||||
Slice internal_key() const { return Slice(rep_); }
|
||||
|
||||
void SetFrom(const ParsedInternalKey& p) {
|
||||
rep_.clear();
|
||||
@@ -253,12 +181,8 @@ inline bool ParseInternalKey(const Slice& internal_key,
|
||||
unsigned char c = num & 0xff;
|
||||
result->sequence = num >> 8;
|
||||
result->type = static_cast<ValueType>(c);
|
||||
if (IsExpiryKey((ValueType)c))
|
||||
result->expiry=DecodeFixed64(internal_key.data() + n - KeySuffixSize((ValueType)c));
|
||||
else
|
||||
result->expiry=0;
|
||||
result->user_key = Slice(internal_key.data(), n - KeySuffixSize((ValueType)c));
|
||||
return (c <= static_cast<unsigned char>(kTypeValueExplicitExpiry));
|
||||
result->user_key = Slice(internal_key.data(), n - 8);
|
||||
return (c <= static_cast<unsigned char>(kTypeValue));
|
||||
}
|
||||
|
||||
// A helper class useful for DBImpl::Get()
|
||||
@@ -266,7 +190,7 @@ class LookupKey {
|
||||
public:
|
||||
// Initialize *this for looking up user_key at a snapshot with
|
||||
// the specified sequence number.
|
||||
LookupKey(const Slice& user_key, SequenceNumber sequence, KeyMetaData * meta=NULL);
|
||||
LookupKey(const Slice& user_key, SequenceNumber sequence);
|
||||
|
||||
~LookupKey();
|
||||
|
||||
@@ -277,38 +201,12 @@ class LookupKey {
|
||||
Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
|
||||
|
||||
// Return the user key
|
||||
Slice user_key() const
|
||||
{ return Slice(kstart_, end_ - kstart_ - KeySuffixSize(internal_key())); }
|
||||
|
||||
// did requestor have metadata object?
|
||||
bool WantsKeyMetaData() const {return(NULL!=meta_);};
|
||||
|
||||
void SetKeyMetaData(ValueType type, SequenceNumber seq, ExpiryTimeMicros expiry) const
|
||||
{if (NULL!=meta_)
|
||||
{
|
||||
meta_->m_Type=type;
|
||||
meta_->m_Sequence=seq;
|
||||
meta_->m_Expiry=expiry;
|
||||
} // if
|
||||
};
|
||||
|
||||
void SetKeyMetaData(const ParsedInternalKey & pi_key) const
|
||||
{if (NULL!=meta_)
|
||||
{
|
||||
meta_->m_Type=pi_key.type;
|
||||
meta_->m_Sequence=pi_key.sequence;
|
||||
meta_->m_Expiry=pi_key.expiry;
|
||||
} // if
|
||||
};
|
||||
|
||||
void SetKeyMetaData(const KeyMetaData & meta) const
|
||||
{if (NULL!=meta_) *meta_=meta;};
|
||||
Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
|
||||
|
||||
private:
|
||||
// We construct a char array of the form:
|
||||
// klength varint32 <-- start_
|
||||
// userkey char[klength] <-- kstart_
|
||||
// optional uint64
|
||||
// tag uint64
|
||||
// <-- end_
|
||||
// The array is a suitable MemTable key.
|
||||
@@ -318,9 +216,6 @@ class LookupKey {
|
||||
const char* end_;
|
||||
char space_[200]; // Avoid allocation for short keys
|
||||
|
||||
// allow code that finds the key to place metadata here, even if 'const'
|
||||
mutable KeyMetaData * meta_;
|
||||
|
||||
// No copying allowed
|
||||
LookupKey(const LookupKey&);
|
||||
void operator=(const LookupKey&);
|
||||
@@ -328,47 +223,8 @@ class LookupKey {
|
||||
|
||||
inline LookupKey::~LookupKey() {
|
||||
if (start_ != space_) delete[] start_;
|
||||
};
|
||||
|
||||
|
||||
// this class was constructed from code with DBImpl::DoCompactionWork (db_impl.cc)
|
||||
// so it could be shared within BuildTable (and thus reduce Level 0 bloating)
|
||||
class KeyRetirement
|
||||
{
|
||||
protected:
|
||||
// "state" from previous key reviewed
|
||||
std::string current_user_key;
|
||||
bool has_current_user_key;
|
||||
SequenceNumber last_sequence_for_key;
|
||||
|
||||
// database values needed for processing
|
||||
const Comparator * user_comparator;
|
||||
SequenceNumber smallest_snapshot;
|
||||
const Options * options;
|
||||
Compaction * const compaction;
|
||||
|
||||
bool valid;
|
||||
size_t dropped; // tombstone or old version dropped
|
||||
size_t expired; // expired dropped
|
||||
|
||||
public:
|
||||
KeyRetirement(const Comparator * UserComparator, SequenceNumber SmallestSnapshot,
|
||||
const Options * Opts, Compaction * const Compaction=NULL);
|
||||
|
||||
virtual ~KeyRetirement();
|
||||
|
||||
bool operator()(Slice & key);
|
||||
|
||||
size_t GetDroppedCount() const {return(dropped);};
|
||||
size_t GetExpiredCount() const {return(expired);};
|
||||
|
||||
private:
|
||||
KeyRetirement();
|
||||
KeyRetirement(const KeyRetirement &);
|
||||
const KeyRetirement & operator=(const KeyRetirement &);
|
||||
|
||||
}; // class KeyRetirement
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
@@ -9,11 +9,10 @@
|
||||
namespace leveldb {
|
||||
|
||||
static std::string IKey(const std::string& user_key,
|
||||
ExpiryTimeMicros exp,
|
||||
uint64_t seq,
|
||||
ValueType vt) {
|
||||
std::string encoded;
|
||||
AppendInternalKey(&encoded, ParsedInternalKey(user_key, exp, seq, vt));
|
||||
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
||||
return encoded;
|
||||
}
|
||||
|
||||
@@ -30,13 +29,12 @@ static std::string ShortSuccessor(const std::string& s) {
|
||||
}
|
||||
|
||||
static void TestKey(const std::string& key,
|
||||
ExpiryTimeMicros exp,
|
||||
uint64_t seq,
|
||||
ValueType vt) {
|
||||
std::string encoded = IKey(key, exp, seq, vt);
|
||||
std::string encoded = IKey(key, seq, vt);
|
||||
|
||||
Slice in(encoded);
|
||||
ParsedInternalKey decoded("", 0, 0, kTypeValue);
|
||||
ParsedInternalKey decoded("", 0, kTypeValue);
|
||||
|
||||
ASSERT_TRUE(ParseInternalKey(in, &decoded));
|
||||
ASSERT_EQ(key, decoded.user_key.ToString());
|
||||
@@ -58,53 +56,53 @@ TEST(FormatTest, InternalKey_EncodeDecode) {
|
||||
};
|
||||
for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
||||
for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
||||
TestKey(keys[k], 0, seq[s], kTypeValue);
|
||||
TestKey("hello", 0, 1, kTypeDeletion);
|
||||
TestKey(keys[k], seq[s], kTypeValue);
|
||||
TestKey("hello", 1, kTypeDeletion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FormatTest, InternalKeyShortSeparator) {
|
||||
// When user keys are same
|
||||
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("foo", 0, 99, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("foo", 0, 101, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("foo", 0, 100, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("foo", 0, 100, kTypeDeletion)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 99, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 101, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 100, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 100, kTypeDeletion)));
|
||||
|
||||
// When user keys are misordered
|
||||
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("bar", 0, 99, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("bar", 99, kTypeValue)));
|
||||
|
||||
// When user keys are different, but correctly ordered
|
||||
ASSERT_EQ(IKey("g", 0, kMaxSequenceNumber, kValueTypeForSeek),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("hello", 0, 200, kTypeValue)));
|
||||
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("hello", 200, kTypeValue)));
|
||||
|
||||
// When start user key is prefix of limit user key
|
||||
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foo", 0, 100, kTypeValue),
|
||||
IKey("foobar", 0, 200, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foobar", 200, kTypeValue)));
|
||||
|
||||
// When limit user key is prefix of start user key
|
||||
ASSERT_EQ(IKey("foobar", 0, 100, kTypeValue),
|
||||
Shorten(IKey("foobar", 0, 100, kTypeValue),
|
||||
IKey("foo", 0, 200, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foobar", 100, kTypeValue),
|
||||
Shorten(IKey("foobar", 100, kTypeValue),
|
||||
IKey("foo", 200, kTypeValue)));
|
||||
}
|
||||
|
||||
TEST(FormatTest, InternalKeyShortestSuccessor) {
|
||||
ASSERT_EQ(IKey("g", 0, kMaxSequenceNumber, kValueTypeForSeek),
|
||||
ShortSuccessor(IKey("foo", 0, 100, kTypeValue)));
|
||||
ASSERT_EQ(IKey("\xff\xff", 0, 100, kTypeValue),
|
||||
ShortSuccessor(IKey("\xff\xff", 0, 100, kTypeValue)));
|
||||
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
||||
ShortSuccessor(IKey("foo", 100, kTypeValue)));
|
||||
ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue),
|
||||
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
@@ -35,93 +35,112 @@ bool GuessType(const std::string& fname, FileType* type) {
|
||||
// Notified when log reader encounters corruption.
|
||||
class CorruptionReporter : public log::Reader::Reporter {
|
||||
public:
|
||||
WritableFile* dst_;
|
||||
virtual void Corruption(size_t bytes, const Status& status) {
|
||||
printf("corruption: %d bytes; %s\n",
|
||||
static_cast<int>(bytes),
|
||||
status.ToString().c_str());
|
||||
std::string r = "corruption: ";
|
||||
AppendNumberTo(&r, bytes);
|
||||
r += " bytes; ";
|
||||
r += status.ToString();
|
||||
r.push_back('\n');
|
||||
dst_->Append(r);
|
||||
}
|
||||
};
|
||||
|
||||
// Print contents of a log file. (*func)() is called on every record.
|
||||
bool PrintLogContents(Env* env, const std::string& fname,
|
||||
void (*func)(Slice)) {
|
||||
Status PrintLogContents(Env* env, const std::string& fname,
|
||||
void (*func)(uint64_t, Slice, WritableFile*),
|
||||
WritableFile* dst) {
|
||||
SequentialFile* file;
|
||||
Status s = env->NewSequentialFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
return false;
|
||||
return s;
|
||||
}
|
||||
CorruptionReporter reporter;
|
||||
reporter.dst_ = dst;
|
||||
log::Reader reader(file, &reporter, true, 0);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
printf("--- offset %llu; ",
|
||||
static_cast<unsigned long long>(reader.LastRecordOffset()));
|
||||
(*func)(record);
|
||||
(*func)(reader.LastRecordOffset(), record, dst);
|
||||
}
|
||||
delete file;
|
||||
return true;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Called on every item found in a WriteBatch.
|
||||
class WriteBatchItemPrinter : public WriteBatch::Handler {
|
||||
public:
|
||||
uint64_t offset_;
|
||||
uint64_t sequence_;
|
||||
|
||||
WritableFile* dst_;
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
printf(" put '%s' '%s'\n",
|
||||
EscapeString(key).c_str(),
|
||||
EscapeString(value).c_str());
|
||||
std::string r = " put '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "' '";
|
||||
AppendEscapedStringTo(&r, value);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
printf(" del '%s'\n",
|
||||
EscapeString(key).c_str());
|
||||
std::string r = " del '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kLogFile.
|
||||
static void WriteBatchPrinter(Slice record) {
|
||||
static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
std::string r = "--- offset ";
|
||||
AppendNumberTo(&r, pos);
|
||||
r += "; ";
|
||||
if (record.size() < 12) {
|
||||
printf("log record length %d is too small\n",
|
||||
static_cast<int>(record.size()));
|
||||
r += "log record length ";
|
||||
AppendNumberTo(&r, record.size());
|
||||
r += " is too small\n";
|
||||
dst->Append(r);
|
||||
return;
|
||||
}
|
||||
WriteBatch batch;
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
printf("sequence %llu\n",
|
||||
static_cast<unsigned long long>(WriteBatchInternal::Sequence(&batch)));
|
||||
r += "sequence ";
|
||||
AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch));
|
||||
r.push_back('\n');
|
||||
dst->Append(r);
|
||||
WriteBatchItemPrinter batch_item_printer;
|
||||
batch_item_printer.dst_ = dst;
|
||||
Status s = batch.Iterate(&batch_item_printer);
|
||||
if (!s.ok()) {
|
||||
printf(" error: %s\n", s.ToString().c_str());
|
||||
dst->Append(" error: " + s.ToString() + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
bool DumpLog(Env* env, const std::string& fname) {
|
||||
return PrintLogContents(env, fname, WriteBatchPrinter);
|
||||
Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return PrintLogContents(env, fname, WriteBatchPrinter, dst);
|
||||
}
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kDescriptorFile.
|
||||
static void VersionEditPrinter(Slice record) {
|
||||
static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
std::string r = "--- offset ";
|
||||
AppendNumberTo(&r, pos);
|
||||
r += "; ";
|
||||
VersionEdit edit;
|
||||
Status s = edit.DecodeFrom(record);
|
||||
if (!s.ok()) {
|
||||
printf("%s\n", s.ToString().c_str());
|
||||
return;
|
||||
r += s.ToString();
|
||||
r.push_back('\n');
|
||||
} else {
|
||||
r += edit.DebugString();
|
||||
}
|
||||
printf("%s", edit.DebugString().c_str());
|
||||
dst->Append(r);
|
||||
}
|
||||
|
||||
bool DumpDescriptor(Env* env, const std::string& fname) {
|
||||
return PrintLogContents(env, fname, VersionEditPrinter);
|
||||
Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return PrintLogContents(env, fname, VersionEditPrinter, dst);
|
||||
}
|
||||
|
||||
bool DumpTable(Env* env, const std::string& fname) {
|
||||
Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
uint64_t file_size;
|
||||
RandomAccessFile* file = NULL;
|
||||
Table* table = NULL;
|
||||
@@ -137,102 +156,70 @@ bool DumpTable(Env* env, const std::string& fname) {
|
||||
s = Table::Open(Options(), file, file_size, &table);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
delete table;
|
||||
delete file;
|
||||
return false;
|
||||
return s;
|
||||
}
|
||||
|
||||
ReadOptions ro;
|
||||
ro.fill_cache = false;
|
||||
Iterator* iter = table->NewIterator(ro);
|
||||
std::string r;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
r.clear();
|
||||
ParsedInternalKey key;
|
||||
if (!ParseInternalKey(iter->key(), &key)) {
|
||||
printf("badkey '%s' => '%s'\n",
|
||||
EscapeString(iter->key()).c_str(),
|
||||
EscapeString(iter->value()).c_str());
|
||||
r = "badkey '";
|
||||
AppendEscapedStringTo(&r, iter->key());
|
||||
r += "' => '";
|
||||
AppendEscapedStringTo(&r, iter->value());
|
||||
r += "'\n";
|
||||
dst->Append(r);
|
||||
} else {
|
||||
char kbuf[20];
|
||||
const char* type;
|
||||
r = "'";
|
||||
AppendEscapedStringTo(&r, key.user_key);
|
||||
r += "' @ ";
|
||||
AppendNumberTo(&r, key.sequence);
|
||||
r += " : ";
|
||||
if (key.type == kTypeDeletion) {
|
||||
type = "del";
|
||||
r += "del";
|
||||
} else if (key.type == kTypeValue) {
|
||||
type = "val";
|
||||
r += "val";
|
||||
} else {
|
||||
snprintf(kbuf, sizeof(kbuf), "%d", static_cast<int>(key.type));
|
||||
type = kbuf;
|
||||
AppendNumberTo(&r, key.type);
|
||||
}
|
||||
printf("'%s' @ %8llu : %s => '%s'\n",
|
||||
EscapeString(key.user_key).c_str(),
|
||||
static_cast<unsigned long long>(key.sequence),
|
||||
type,
|
||||
EscapeString(iter->value()).c_str());
|
||||
r += " => '";
|
||||
AppendEscapedStringTo(&r, iter->value());
|
||||
r += "'\n";
|
||||
dst->Append(r);
|
||||
}
|
||||
}
|
||||
s = iter->status();
|
||||
if (!s.ok()) {
|
||||
printf("iterator error: %s\n", s.ToString().c_str());
|
||||
dst->Append("iterator error: " + s.ToString() + "\n");
|
||||
}
|
||||
|
||||
delete iter;
|
||||
delete table;
|
||||
delete file;
|
||||
return true;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool DumpFile(Env* env, const std::string& fname) {
|
||||
} // namespace
|
||||
|
||||
Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
FileType ftype;
|
||||
if (!GuessType(fname, &ftype)) {
|
||||
fprintf(stderr, "%s: unknown file type\n", fname.c_str());
|
||||
return false;
|
||||
return Status::InvalidArgument(fname + ": unknown file type");
|
||||
}
|
||||
switch (ftype) {
|
||||
case kLogFile: return DumpLog(env, fname);
|
||||
case kDescriptorFile: return DumpDescriptor(env, fname);
|
||||
case kTableFile: return DumpTable(env, fname);
|
||||
|
||||
default: {
|
||||
fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str());
|
||||
case kLogFile: return DumpLog(env, fname, dst);
|
||||
case kDescriptorFile: return DumpDescriptor(env, fname, dst);
|
||||
case kTableFile: return DumpTable(env, fname, dst);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return Status::InvalidArgument(fname + ": not a dump-able file type");
|
||||
}
|
||||
|
||||
bool HandleDumpCommand(Env* env, char** files, int num) {
|
||||
bool ok = true;
|
||||
for (int i = 0; i < num; i++) {
|
||||
ok &= DumpFile(env, files[i]);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
}
|
||||
} // namespace leveldb
|
||||
|
||||
static void Usage() {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Usage: leveldbutil command...\n"
|
||||
" dump files... -- dump contents of specified files\n"
|
||||
);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
leveldb::Env* env = leveldb::Env::Default();
|
||||
bool ok = true;
|
||||
if (argc < 2) {
|
||||
Usage();
|
||||
ok = false;
|
||||
} else {
|
||||
std::string command = argv[1];
|
||||
if (command == "dump") {
|
||||
ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
|
||||
} else {
|
||||
Usage();
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
return (ok ? 0 : 1);
|
||||
}
|
||||
@@ -0,0 +1,554 @@
|
||||
// Copyright 2014 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
// This test uses a custom Env to keep track of the state of a filesystem as of
|
||||
// the last "sync". It then checks for data loss errors by purposely dropping
|
||||
// file data (or entire files) not protected by a "sync".
|
||||
|
||||
#include "leveldb/db.h"
|
||||
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_format.h"
|
||||
#include "db/version_set.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
static const int kValueSize = 1000;
|
||||
static const int kMaxNumValues = 2000;
|
||||
static const size_t kNumIterations = 3;
|
||||
|
||||
class FaultInjectionTestEnv;
|
||||
|
||||
namespace {
|
||||
|
||||
// Assume a filename, and not a directory name like "/foo/bar/"
|
||||
static std::string GetDirName(const std::string filename) {
|
||||
size_t found = filename.find_last_of("/\\");
|
||||
if (found == std::string::npos) {
|
||||
return "";
|
||||
} else {
|
||||
return filename.substr(0, found);
|
||||
}
|
||||
}
|
||||
|
||||
Status SyncDir(const std::string& dir) {
|
||||
// As this is a test it isn't required to *actually* sync this directory.
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// A basic file truncation function suitable for this test.
|
||||
Status Truncate(const std::string& filename, uint64_t length) {
|
||||
leveldb::Env* env = leveldb::Env::Default();
|
||||
|
||||
SequentialFile* orig_file;
|
||||
Status s = env->NewSequentialFile(filename, &orig_file);
|
||||
if (!s.ok())
|
||||
return s;
|
||||
|
||||
char* scratch = new char[length];
|
||||
leveldb::Slice result;
|
||||
s = orig_file->Read(length, &result, scratch);
|
||||
delete orig_file;
|
||||
if (s.ok()) {
|
||||
std::string tmp_name = GetDirName(filename) + "/truncate.tmp";
|
||||
WritableFile* tmp_file;
|
||||
s = env->NewWritableFile(tmp_name, &tmp_file);
|
||||
if (s.ok()) {
|
||||
s = tmp_file->Append(result);
|
||||
delete tmp_file;
|
||||
if (s.ok()) {
|
||||
s = env->RenameFile(tmp_name, filename);
|
||||
} else {
|
||||
env->DeleteFile(tmp_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
delete[] scratch;
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
struct FileState {
|
||||
std::string filename_;
|
||||
ssize_t pos_;
|
||||
ssize_t pos_at_last_sync_;
|
||||
ssize_t pos_at_last_flush_;
|
||||
|
||||
FileState(const std::string& filename)
|
||||
: filename_(filename),
|
||||
pos_(-1),
|
||||
pos_at_last_sync_(-1),
|
||||
pos_at_last_flush_(-1) { }
|
||||
|
||||
FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
|
||||
|
||||
bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; }
|
||||
|
||||
Status DropUnsyncedData() const;
|
||||
};
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// A wrapper around WritableFile which informs another Env whenever this file
|
||||
// is written to or sync'ed.
|
||||
class TestWritableFile : public WritableFile {
|
||||
public:
|
||||
TestWritableFile(const FileState& state,
|
||||
WritableFile* f,
|
||||
FaultInjectionTestEnv* env);
|
||||
virtual ~TestWritableFile();
|
||||
virtual Status Append(const Slice& data);
|
||||
virtual Status Close();
|
||||
virtual Status Flush();
|
||||
virtual Status Sync();
|
||||
|
||||
private:
|
||||
FileState state_;
|
||||
WritableFile* target_;
|
||||
bool writable_file_opened_;
|
||||
FaultInjectionTestEnv* env_;
|
||||
|
||||
Status SyncParent();
|
||||
};
|
||||
|
||||
class FaultInjectionTestEnv : public EnvWrapper {
|
||||
public:
|
||||
FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {}
|
||||
virtual ~FaultInjectionTestEnv() { }
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result);
|
||||
virtual Status NewAppendableFile(const std::string& fname,
|
||||
WritableFile** result);
|
||||
virtual Status DeleteFile(const std::string& f);
|
||||
virtual Status RenameFile(const std::string& s, const std::string& t);
|
||||
|
||||
void WritableFileClosed(const FileState& state);
|
||||
Status DropUnsyncedFileData();
|
||||
Status DeleteFilesCreatedAfterLastDirSync();
|
||||
void DirWasSynced();
|
||||
bool IsFileCreatedSinceLastDirSync(const std::string& filename);
|
||||
void ResetState();
|
||||
void UntrackFile(const std::string& f);
|
||||
// Setting the filesystem to inactive is the test equivalent to simulating a
|
||||
// system reset. Setting to inactive will freeze our saved filesystem state so
|
||||
// that it will stop being recorded. It can then be reset back to the state at
|
||||
// the time of the reset.
|
||||
bool IsFilesystemActive() const { return filesystem_active_; }
|
||||
void SetFilesystemActive(bool active) { filesystem_active_ = active; }
|
||||
|
||||
private:
|
||||
port::Mutex mutex_;
|
||||
std::map<std::string, FileState> db_file_state_;
|
||||
std::set<std::string> new_files_since_last_dir_sync_;
|
||||
bool filesystem_active_; // Record flushes, syncs, writes
|
||||
};
|
||||
|
||||
TestWritableFile::TestWritableFile(const FileState& state,
|
||||
WritableFile* f,
|
||||
FaultInjectionTestEnv* env)
|
||||
: state_(state),
|
||||
target_(f),
|
||||
writable_file_opened_(true),
|
||||
env_(env) {
|
||||
assert(f != NULL);
|
||||
}
|
||||
|
||||
TestWritableFile::~TestWritableFile() {
|
||||
if (writable_file_opened_) {
|
||||
Close();
|
||||
}
|
||||
delete target_;
|
||||
}
|
||||
|
||||
Status TestWritableFile::Append(const Slice& data) {
|
||||
Status s = target_->Append(data);
|
||||
if (s.ok() && env_->IsFilesystemActive()) {
|
||||
state_.pos_ += data.size();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status TestWritableFile::Close() {
|
||||
writable_file_opened_ = false;
|
||||
Status s = target_->Close();
|
||||
if (s.ok()) {
|
||||
env_->WritableFileClosed(state_);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status TestWritableFile::Flush() {
|
||||
Status s = target_->Flush();
|
||||
if (s.ok() && env_->IsFilesystemActive()) {
|
||||
state_.pos_at_last_flush_ = state_.pos_;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status TestWritableFile::SyncParent() {
|
||||
Status s = SyncDir(GetDirName(state_.filename_));
|
||||
if (s.ok()) {
|
||||
env_->DirWasSynced();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status TestWritableFile::Sync() {
|
||||
if (!env_->IsFilesystemActive()) {
|
||||
return Status::OK();
|
||||
}
|
||||
// Ensure new files referred to by the manifest are in the filesystem.
|
||||
Status s = target_->Sync();
|
||||
if (s.ok()) {
|
||||
state_.pos_at_last_sync_ = state_.pos_;
|
||||
}
|
||||
if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) {
|
||||
Status ps = SyncParent();
|
||||
if (s.ok() && !ps.ok()) {
|
||||
s = ps;
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
|
||||
WritableFile** result) {
|
||||
WritableFile* actual_writable_file;
|
||||
Status s = target()->NewWritableFile(fname, &actual_writable_file);
|
||||
if (s.ok()) {
|
||||
FileState state(fname);
|
||||
state.pos_ = 0;
|
||||
*result = new TestWritableFile(state, actual_writable_file, this);
|
||||
// NewWritableFile doesn't append to files, so if the same file is
|
||||
// opened again then it will be truncated - so forget our saved
|
||||
// state.
|
||||
UntrackFile(fname);
|
||||
MutexLock l(&mutex_);
|
||||
new_files_since_last_dir_sync_.insert(fname);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
|
||||
WritableFile** result) {
|
||||
WritableFile* actual_writable_file;
|
||||
Status s = target()->NewAppendableFile(fname, &actual_writable_file);
|
||||
if (s.ok()) {
|
||||
FileState state(fname);
|
||||
state.pos_ = 0;
|
||||
{
|
||||
MutexLock l(&mutex_);
|
||||
if (db_file_state_.count(fname) == 0) {
|
||||
new_files_since_last_dir_sync_.insert(fname);
|
||||
} else {
|
||||
state = db_file_state_[fname];
|
||||
}
|
||||
}
|
||||
*result = new TestWritableFile(state, actual_writable_file, this);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::DropUnsyncedFileData() {
|
||||
Status s;
|
||||
MutexLock l(&mutex_);
|
||||
for (std::map<std::string, FileState>::const_iterator it =
|
||||
db_file_state_.begin();
|
||||
s.ok() && it != db_file_state_.end(); ++it) {
|
||||
const FileState& state = it->second;
|
||||
if (!state.IsFullySynced()) {
|
||||
s = state.DropUnsyncedData();
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
void FaultInjectionTestEnv::DirWasSynced() {
|
||||
MutexLock l(&mutex_);
|
||||
new_files_since_last_dir_sync_.clear();
|
||||
}
|
||||
|
||||
bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync(
|
||||
const std::string& filename) {
|
||||
MutexLock l(&mutex_);
|
||||
return new_files_since_last_dir_sync_.find(filename) !=
|
||||
new_files_since_last_dir_sync_.end();
|
||||
}
|
||||
|
||||
void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
|
||||
MutexLock l(&mutex_);
|
||||
db_file_state_.erase(f);
|
||||
new_files_since_last_dir_sync_.erase(f);
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
|
||||
Status s = EnvWrapper::DeleteFile(f);
|
||||
ASSERT_OK(s);
|
||||
if (s.ok()) {
|
||||
UntrackFile(f);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::RenameFile(const std::string& s,
|
||||
const std::string& t) {
|
||||
Status ret = EnvWrapper::RenameFile(s, t);
|
||||
|
||||
if (ret.ok()) {
|
||||
MutexLock l(&mutex_);
|
||||
if (db_file_state_.find(s) != db_file_state_.end()) {
|
||||
db_file_state_[t] = db_file_state_[s];
|
||||
db_file_state_.erase(s);
|
||||
}
|
||||
|
||||
if (new_files_since_last_dir_sync_.erase(s) != 0) {
|
||||
assert(new_files_since_last_dir_sync_.find(t) ==
|
||||
new_files_since_last_dir_sync_.end());
|
||||
new_files_since_last_dir_sync_.insert(t);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void FaultInjectionTestEnv::ResetState() {
|
||||
// Since we are not destroying the database, the existing files
|
||||
// should keep their recorded synced/flushed state. Therefore
|
||||
// we do not reset db_file_state_ and new_files_since_last_dir_sync_.
|
||||
MutexLock l(&mutex_);
|
||||
SetFilesystemActive(true);
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
|
||||
// Because DeleteFile access this container make a copy to avoid deadlock
|
||||
mutex_.Lock();
|
||||
std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
|
||||
new_files_since_last_dir_sync_.end());
|
||||
mutex_.Unlock();
|
||||
Status s;
|
||||
std::set<std::string>::const_iterator it;
|
||||
for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) {
|
||||
s = DeleteFile(*it);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
|
||||
MutexLock l(&mutex_);
|
||||
db_file_state_[state.filename_] = state;
|
||||
}
|
||||
|
||||
Status FileState::DropUnsyncedData() const {
|
||||
ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
|
||||
return Truncate(filename_, sync_pos);
|
||||
}
|
||||
|
||||
class FaultInjectionTest {
|
||||
public:
|
||||
enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR };
|
||||
enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES };
|
||||
|
||||
FaultInjectionTestEnv* env_;
|
||||
std::string dbname_;
|
||||
Cache* tiny_cache_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
|
||||
FaultInjectionTest()
|
||||
: env_(new FaultInjectionTestEnv),
|
||||
tiny_cache_(NewLRUCache(100)),
|
||||
db_(NULL) {
|
||||
dbname_ = test::TmpDir() + "/fault_test";
|
||||
DestroyDB(dbname_, Options()); // Destroy any db from earlier run
|
||||
options_.reuse_logs = true;
|
||||
options_.env = env_;
|
||||
options_.paranoid_checks = true;
|
||||
options_.block_cache = tiny_cache_;
|
||||
options_.create_if_missing = true;
|
||||
}
|
||||
|
||||
~FaultInjectionTest() {
|
||||
CloseDB();
|
||||
DestroyDB(dbname_, Options());
|
||||
delete tiny_cache_;
|
||||
delete env_;
|
||||
}
|
||||
|
||||
void ReuseLogs(bool reuse) {
|
||||
options_.reuse_logs = reuse;
|
||||
}
|
||||
|
||||
void Build(int start_idx, int num_vals) {
|
||||
std::string key_space, value_space;
|
||||
WriteBatch batch;
|
||||
for (int i = start_idx; i < start_idx + num_vals; i++) {
|
||||
Slice key = Key(i, &key_space);
|
||||
batch.Clear();
|
||||
batch.Put(key, Value(i, &value_space));
|
||||
WriteOptions options;
|
||||
ASSERT_OK(db_->Write(options, &batch));
|
||||
}
|
||||
}
|
||||
|
||||
Status ReadValue(int i, std::string* val) const {
|
||||
std::string key_space, value_space;
|
||||
Slice key = Key(i, &key_space);
|
||||
Value(i, &value_space);
|
||||
ReadOptions options;
|
||||
return db_->Get(options, key, val);
|
||||
}
|
||||
|
||||
Status Verify(int start_idx, int num_vals,
|
||||
ExpectedVerifResult expected) const {
|
||||
std::string val;
|
||||
std::string value_space;
|
||||
Status s;
|
||||
for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) {
|
||||
Value(i, &value_space);
|
||||
s = ReadValue(i, &val);
|
||||
if (expected == VAL_EXPECT_NO_ERROR) {
|
||||
if (s.ok()) {
|
||||
ASSERT_EQ(value_space, val);
|
||||
}
|
||||
} else if (s.ok()) {
|
||||
fprintf(stderr, "Expected an error at %d, but was OK\n", i);
|
||||
s = Status::IOError(dbname_, "Expected value error:");
|
||||
} else {
|
||||
s = Status::OK(); // An expected error
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
// Return the ith key
|
||||
Slice Key(int i, std::string* storage) const {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "%016d", i);
|
||||
storage->assign(buf, strlen(buf));
|
||||
return Slice(*storage);
|
||||
}
|
||||
|
||||
// Return the value to associate with the specified key
|
||||
Slice Value(int k, std::string* storage) const {
|
||||
Random r(k);
|
||||
return test::RandomString(&r, kValueSize, storage);
|
||||
}
|
||||
|
||||
Status OpenDB() {
|
||||
delete db_;
|
||||
db_ = NULL;
|
||||
env_->ResetState();
|
||||
return DB::Open(options_, dbname_, &db_);
|
||||
}
|
||||
|
||||
void CloseDB() {
|
||||
delete db_;
|
||||
db_ = NULL;
|
||||
}
|
||||
|
||||
void DeleteAllData() {
|
||||
Iterator* iter = db_->NewIterator(ReadOptions());
|
||||
WriteOptions options;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ASSERT_OK(db_->Delete(WriteOptions(), iter->key()));
|
||||
}
|
||||
|
||||
delete iter;
|
||||
}
|
||||
|
||||
void ResetDBState(ResetMethod reset_method) {
|
||||
switch (reset_method) {
|
||||
case RESET_DROP_UNSYNCED_DATA:
|
||||
ASSERT_OK(env_->DropUnsyncedFileData());
|
||||
break;
|
||||
case RESET_DELETE_UNSYNCED_FILES:
|
||||
ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync());
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) {
|
||||
DeleteAllData();
|
||||
Build(0, num_pre_sync);
|
||||
db_->CompactRange(NULL, NULL);
|
||||
Build(num_pre_sync, num_post_sync);
|
||||
}
|
||||
|
||||
void PartialCompactTestReopenWithFault(ResetMethod reset_method,
|
||||
int num_pre_sync,
|
||||
int num_post_sync) {
|
||||
env_->SetFilesystemActive(false);
|
||||
CloseDB();
|
||||
ResetDBState(reset_method);
|
||||
ASSERT_OK(OpenDB());
|
||||
ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
|
||||
ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR));
|
||||
}
|
||||
|
||||
void NoWriteTestPreFault() {
|
||||
}
|
||||
|
||||
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
|
||||
CloseDB();
|
||||
ResetDBState(reset_method);
|
||||
ASSERT_OK(OpenDB());
|
||||
}
|
||||
|
||||
void DoTest() {
|
||||
Random rnd(0);
|
||||
ASSERT_OK(OpenDB());
|
||||
for (size_t idx = 0; idx < kNumIterations; idx++) {
|
||||
int num_pre_sync = rnd.Uniform(kMaxNumValues);
|
||||
int num_post_sync = rnd.Uniform(kMaxNumValues);
|
||||
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync);
|
||||
PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA,
|
||||
num_pre_sync,
|
||||
num_post_sync);
|
||||
|
||||
NoWriteTestPreFault();
|
||||
NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA);
|
||||
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync);
|
||||
// No new files created so we expect all values since no files will be
|
||||
// dropped.
|
||||
PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
|
||||
num_pre_sync + num_post_sync,
|
||||
0);
|
||||
|
||||
NoWriteTestPreFault();
|
||||
NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
TEST(FaultInjectionTest, FaultTestNoLogReuse) {
|
||||
ReuseLogs(false);
|
||||
DoTest();
|
||||
}
|
||||
|
||||
TEST(FaultInjectionTest, FaultTestWithLogReuse) {
|
||||
ReuseLogs(true);
|
||||
DoTest();
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+9
-174
@@ -4,14 +4,9 @@
|
||||
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include "db/filename.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/version_set.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace leveldb {
|
||||
@@ -29,50 +24,19 @@ static std::string MakeFileName(const std::string& name, uint64_t number,
|
||||
return name + buf;
|
||||
}
|
||||
|
||||
static std::string MakeFileName2(const Options & options, uint64_t number,
|
||||
int level, const char* suffix) {
|
||||
char buf[100];
|
||||
if (0<=level)
|
||||
snprintf(buf, sizeof(buf), "/%s_%-d/%06llu.%s",
|
||||
suffix, level,
|
||||
static_cast<unsigned long long>(number),
|
||||
suffix);
|
||||
else if (-1==level)
|
||||
snprintf(buf, sizeof(buf), "/%s/%06llu.%s",
|
||||
suffix,
|
||||
static_cast<unsigned long long>(number),
|
||||
suffix);
|
||||
else if (-2==level)
|
||||
snprintf(buf, sizeof(buf), "/%06llu.%s",
|
||||
static_cast<unsigned long long>(number),
|
||||
suffix);
|
||||
|
||||
return((level<(int)options.tiered_slow_level ?
|
||||
options.tiered_fast_prefix : options.tiered_slow_prefix) + buf);
|
||||
}
|
||||
|
||||
std::string MakeDirName2(const Options & options,
|
||||
int level, const char* suffix) {
|
||||
char buf[100];
|
||||
if (-1!=level)
|
||||
snprintf(buf, sizeof(buf), "/%s_%-d",
|
||||
suffix, level);
|
||||
else
|
||||
snprintf(buf, sizeof(buf), "/%s",
|
||||
suffix);
|
||||
|
||||
return((level<(int)options.tiered_slow_level ?
|
||||
options.tiered_fast_prefix : options.tiered_slow_prefix) + buf);
|
||||
}
|
||||
|
||||
std::string LogFileName(const std::string& name, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(name, number, "log");
|
||||
}
|
||||
|
||||
std::string TableFileName(const Options & options, uint64_t number, int level) {
|
||||
std::string TableFileName(const std::string& name, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName2(options, number, level, "sst");
|
||||
return MakeFileName(name, number, "ldb");
|
||||
}
|
||||
|
||||
std::string SSTTableFileName(const std::string& name, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(name, number, "sst");
|
||||
}
|
||||
|
||||
std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
|
||||
@@ -105,36 +69,6 @@ std::string OldInfoLogFileName(const std::string& dbname) {
|
||||
return dbname + "/LOG.old";
|
||||
}
|
||||
|
||||
//
|
||||
std::string CowFileName(const std::string& dbname) {
|
||||
return dbname + "/COW";
|
||||
}
|
||||
|
||||
|
||||
// Append appropriate "backup" string to input path
|
||||
std::string BackupPath(const std::string& dbname, int backup_num) {
|
||||
std::string dirname;
|
||||
|
||||
char buf[100];
|
||||
if (0 != backup_num)
|
||||
snprintf(buf, sizeof(buf), "/backup.%-d", backup_num);
|
||||
else
|
||||
snprintf(buf, sizeof(buf), "/backup");
|
||||
|
||||
return(dbname + buf);
|
||||
}
|
||||
|
||||
|
||||
// update tiered_fast_prefix and tiered_slow_prefix members of
|
||||
// given Options object to point to desired backup path
|
||||
bool SetBackupPaths(Options & options, int backup_num) {
|
||||
|
||||
options.tiered_fast_prefix = BackupPath(options.tiered_fast_prefix, backup_num);
|
||||
options.tiered_slow_prefix = BackupPath(options.tiered_slow_prefix, backup_num);
|
||||
|
||||
return(true);
|
||||
}
|
||||
|
||||
|
||||
// Owned filenames have the form:
|
||||
// dbname/CURRENT
|
||||
@@ -142,8 +76,7 @@ bool SetBackupPaths(Options & options, int backup_num) {
|
||||
// dbname/LOG
|
||||
// dbname/LOG.old
|
||||
// dbname/MANIFEST-[0-9]+
|
||||
// dbname/[0-9]+.(log|sst)
|
||||
// dbname/COW
|
||||
// dbname/[0-9]+.(log|sst|ldb)
|
||||
bool ParseFileName(const std::string& fname,
|
||||
uint64_t* number,
|
||||
FileType* type) {
|
||||
@@ -151,9 +84,6 @@ bool ParseFileName(const std::string& fname,
|
||||
if (rest == "CURRENT") {
|
||||
*number = 0;
|
||||
*type = kCurrentFile;
|
||||
} else if (rest == "COW") {
|
||||
*number = 0;
|
||||
*type = kCacheWarming;
|
||||
} else if (rest == "LOCK") {
|
||||
*number = 0;
|
||||
*type = kDBLockFile;
|
||||
@@ -181,7 +111,7 @@ bool ParseFileName(const std::string& fname,
|
||||
Slice suffix = rest;
|
||||
if (suffix == Slice(".log")) {
|
||||
*type = kLogFile;
|
||||
} else if (suffix == Slice(".sst")) {
|
||||
} else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
|
||||
*type = kTableFile;
|
||||
} else if (suffix == Slice(".dbtmp")) {
|
||||
*type = kTempFile;
|
||||
@@ -211,99 +141,4 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
Status
|
||||
MakeLevelDirectories(Env * env, const Options & options)
|
||||
{
|
||||
Status ret_stat;
|
||||
int level;
|
||||
std::string dirname;
|
||||
|
||||
for (level=0; level<config::kNumLevels && ret_stat.ok(); ++level)
|
||||
{
|
||||
dirname=MakeDirName2(options, level, "sst");
|
||||
|
||||
// ignoring error since no way to tell if "bad" error, or "already exists" error
|
||||
env->CreateDir(dirname.c_str());
|
||||
} // for
|
||||
|
||||
return(ret_stat);
|
||||
|
||||
} // MakeLevelDirectories
|
||||
|
||||
|
||||
bool
|
||||
TestForLevelDirectories(
|
||||
Env * env,
|
||||
const Options & options,
|
||||
Version * version)
|
||||
{
|
||||
bool ret_flag, again;
|
||||
int level;
|
||||
std::string dirname;
|
||||
|
||||
ret_flag=true;
|
||||
again=true;
|
||||
|
||||
// walk backwards, fault will be in higher levels if partial conversion
|
||||
for (level=config::kNumLevels-1; 0<=level && again; --level)
|
||||
{
|
||||
again=false;
|
||||
|
||||
// does directory exist
|
||||
dirname=MakeDirName2(options, level, "sst");
|
||||
ret_flag=env->FileExists(dirname.c_str());
|
||||
|
||||
// do all files exist in level
|
||||
if (ret_flag)
|
||||
{
|
||||
const std::vector<FileMetaData*> & level_files(version->GetFileList(level));
|
||||
std::vector<FileMetaData*>::const_iterator it;
|
||||
std::string table_name;
|
||||
Status s;
|
||||
|
||||
for (it=level_files.begin(); level_files.end()!=it && ret_flag; ++it)
|
||||
{
|
||||
table_name=TableFileName(options, (*it)->number, level);
|
||||
ret_flag=env->FileExists(table_name.c_str());
|
||||
} // for
|
||||
|
||||
again=ret_flag && 0==level_files.size();
|
||||
} // if
|
||||
} // for
|
||||
|
||||
return(ret_flag);
|
||||
|
||||
} // TestForLevelDirectories
|
||||
|
||||
std::string // replacement dbname ... potentially tiered
|
||||
MakeTieredDbname(
|
||||
const std::string & dbname, // input ... original dbname from DBImpl constructor
|
||||
Options & options) // input/output ... writable Options, tiered values changed
|
||||
{
|
||||
// case for "", used with internal calls to DestroyDB
|
||||
if (0==dbname.size() && 0!=options.tiered_fast_prefix.size())
|
||||
{
|
||||
// do NOTHING ... options already initialized
|
||||
} // if
|
||||
else if (0<(int)options.tiered_slow_level && (int)options.tiered_slow_level<config::kNumLevels
|
||||
&& 0!=options.tiered_fast_prefix.size() && 0!=options.tiered_slow_prefix.size())
|
||||
{
|
||||
options.tiered_fast_prefix.append("/");
|
||||
options.tiered_fast_prefix.append(dbname);
|
||||
|
||||
options.tiered_slow_prefix.append("/");
|
||||
options.tiered_slow_prefix.append(dbname);
|
||||
} // else if
|
||||
else
|
||||
{
|
||||
options.tiered_slow_level=0;
|
||||
options.tiered_fast_prefix=dbname; // duplicate as is
|
||||
options.tiered_slow_prefix=dbname;
|
||||
} // else
|
||||
|
||||
return(options.tiered_fast_prefix);
|
||||
|
||||
} // MakeTieredDbname
|
||||
|
||||
} // namespace leveldb
|
||||
@@ -9,7 +9,6 @@
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "port/port.h"
|
||||
@@ -17,7 +16,6 @@
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
class Version;
|
||||
|
||||
enum FileType {
|
||||
kLogFile,
|
||||
@@ -26,24 +24,9 @@ enum FileType {
|
||||
kDescriptorFile,
|
||||
kCurrentFile,
|
||||
kTempFile,
|
||||
kInfoLogFile, // Either the current one, or an old one
|
||||
kCacheWarming
|
||||
kInfoLogFile // Either the current one, or an old one
|
||||
};
|
||||
|
||||
// Riak specific routine to help create sst_? subdirectory names
|
||||
std::string MakeDirName2(const Options & options,
|
||||
int level, const char* suffix);
|
||||
|
||||
// Riak specific routine to help create sst_? subdirectories
|
||||
Status MakeLevelDirectories(Env * env, const Options & options);
|
||||
|
||||
// Riak specific routine to test if sst_? subdirectories exist
|
||||
bool TestForLevelDirectories(Env * env, const Options & options, class Version *);
|
||||
|
||||
// Riak specific routine to standardize conversion of dbname and
|
||||
// Options' tiered directories (options parameter is MODIFIED)
|
||||
std::string MakeTieredDbname(const std::string &dbname, Options & options_rw);
|
||||
|
||||
// Return the name of the log file with the specified number
|
||||
// in the db named by "dbname". The result will be prefixed with
|
||||
// "dbname".
|
||||
@@ -52,8 +35,12 @@ extern std::string LogFileName(const std::string& dbname, uint64_t number);
|
||||
// Return the name of the sstable with the specified number
|
||||
// in the db named by "dbname". The result will be prefixed with
|
||||
// "dbname".
|
||||
extern std::string TableFileName(const Options & options, uint64_t number,
|
||||
int level);
|
||||
extern std::string TableFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the legacy file name for an sstable with the specified number
|
||||
// in the db named by "dbname". The result will be prefixed with
|
||||
// "dbname".
|
||||
extern std::string SSTTableFileName(const std::string& dbname, uint64_t number);
|
||||
|
||||
// Return the name of the descriptor file for the db named by
|
||||
// "dbname" and the specified incarnation number. The result will be
|
||||
@@ -80,21 +67,10 @@ extern std::string InfoLogFileName(const std::string& dbname);
|
||||
// Return the name of the old info log file for "dbname".
|
||||
extern std::string OldInfoLogFileName(const std::string& dbname);
|
||||
|
||||
// Return the name of the cache object file for the db named by
|
||||
// "dbname". The result will be prefixed with "dbname".
|
||||
extern std::string CowFileName(const std::string& dbname);
|
||||
|
||||
// Append appropriate "backup" string to input path
|
||||
extern std::string BackupPath(const std::string& dbname, int backup_num);
|
||||
|
||||
// update tiered_fast_prefix and tiered_slow_prefix members of
|
||||
// given Options object to point to backup path
|
||||
extern bool SetBackupPaths(Options & options, int backup_num);
|
||||
|
||||
// If filename is a leveldb file, store the type of the file in *type.
|
||||
// The number encoded in the filename is stored in *number. If the
|
||||
// filename was successfully parsed, returns true. Else return false.
|
||||
extern bool ParseFileName(const std::string& tiered_filename,
|
||||
extern bool ParseFileName(const std::string& filename,
|
||||
uint64_t* number,
|
||||
FileType* type);
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ TEST(FileNameTest, Parse) {
|
||||
{ "100.log", 100, kLogFile },
|
||||
{ "0.log", 0, kLogFile },
|
||||
{ "0.sst", 0, kTableFile },
|
||||
{ "0.ldb", 0, kTableFile },
|
||||
{ "CURRENT", 0, kCurrentFile },
|
||||
{ "LOCK", 0, kDBLockFile },
|
||||
{ "MANIFEST-2", 2, kDescriptorFile },
|
||||
@@ -70,14 +71,13 @@ TEST(FileNameTest, Parse) {
|
||||
for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
||||
std::string f = errors[i];
|
||||
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FileNameTest, Construction) {
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
std::string fname;
|
||||
Options options;
|
||||
|
||||
fname = CurrentFileName("foo");
|
||||
ASSERT_EQ("foo/", std::string(fname.data(), 4));
|
||||
@@ -97,40 +97,12 @@ TEST(FileNameTest, Construction) {
|
||||
ASSERT_EQ(192, number);
|
||||
ASSERT_EQ(kLogFile, type);
|
||||
|
||||
options.tiered_fast_prefix="bar";
|
||||
options.tiered_slow_prefix="bar";
|
||||
fname = TableFileName(options, 200, 1);
|
||||
fname = TableFileName("bar", 200);
|
||||
ASSERT_EQ("bar/", std::string(fname.data(), 4));
|
||||
ASSERT_EQ("sst_1/", std::string(fname.substr(4,6)));
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 10, &number, &type));
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
|
||||
ASSERT_EQ(200, number);
|
||||
ASSERT_EQ(kTableFile, type);
|
||||
|
||||
fname = TableFileName(options, 400, 4);
|
||||
ASSERT_EQ("bar/", std::string(fname.data(), 4));
|
||||
ASSERT_EQ("sst_4/", std::string(fname.substr(4,6)));
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 10, &number, &type));
|
||||
ASSERT_EQ(400, number);
|
||||
ASSERT_EQ(kTableFile, type);
|
||||
|
||||
options.tiered_slow_level=4;
|
||||
options.tiered_fast_prefix="fast";
|
||||
options.tiered_slow_prefix="slow";
|
||||
fname = TableFileName(options, 500, 3);
|
||||
ASSERT_EQ("fast/", std::string(fname.data(), 5));
|
||||
ASSERT_EQ("sst_3/", std::string(fname.substr(5,6)));
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 11, &number, &type));
|
||||
ASSERT_EQ(500, number);
|
||||
ASSERT_EQ(kTableFile, type);
|
||||
|
||||
fname = TableFileName(options, 600, 4);
|
||||
ASSERT_EQ("slow/", std::string(fname.data(), 5));
|
||||
ASSERT_EQ("sst_4/", std::string(fname.substr(5,6)));
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 11, &number, &type));
|
||||
ASSERT_EQ(600, number);
|
||||
ASSERT_EQ(kTableFile, type);
|
||||
|
||||
|
||||
fname = DescriptorFileName("bar", 100);
|
||||
ASSERT_EQ("bar/", std::string(fname.data(), 4));
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
|
||||
@@ -142,48 +114,6 @@ TEST(FileNameTest, Construction) {
|
||||
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
|
||||
ASSERT_EQ(999, number);
|
||||
ASSERT_EQ(kTempFile, type);
|
||||
|
||||
fname = CowFileName("/what/goes/moo");
|
||||
ASSERT_EQ("/what/goes/moo/COW", fname);
|
||||
|
||||
fname = BackupPath("/var/db/riak/data/leveldb/0",0);
|
||||
ASSERT_EQ("/var/db/riak/data/leveldb/0/backup", fname);
|
||||
|
||||
fname = BackupPath("/var/db/riak/data/leveldb/0",1);
|
||||
ASSERT_EQ("/var/db/riak/data/leveldb/0/backup.1", fname);
|
||||
|
||||
fname = BackupPath("/var/db/riak/data/leveldb/0",5);
|
||||
ASSERT_EQ("/var/db/riak/data/leveldb/0/backup.5", fname);
|
||||
|
||||
options.tiered_slow_level=4;
|
||||
options.tiered_fast_prefix="fast";
|
||||
options.tiered_slow_prefix="slow";
|
||||
fname = SetBackupPaths(options,0);
|
||||
ASSERT_EQ("fast/backup", options.tiered_fast_prefix);
|
||||
ASSERT_EQ("slow/backup", options.tiered_slow_prefix);
|
||||
|
||||
options.tiered_slow_level=4;
|
||||
options.tiered_fast_prefix="fast";
|
||||
options.tiered_slow_prefix="slow";
|
||||
fname = SetBackupPaths(options,3);
|
||||
ASSERT_EQ("fast/backup.3", options.tiered_fast_prefix);
|
||||
ASSERT_EQ("slow/backup.3", options.tiered_slow_prefix);
|
||||
|
||||
|
||||
options.tiered_slow_level=4;
|
||||
options.tiered_fast_prefix="//mnt/fast";
|
||||
options.tiered_slow_prefix="//mnt/slow";
|
||||
fname=MakeTieredDbname("riak/data/leveldb", options);
|
||||
ASSERT_EQ("//mnt/fast/riak/data/leveldb", fname);
|
||||
ASSERT_EQ("//mnt/fast/riak/data/leveldb", options.tiered_fast_prefix);
|
||||
ASSERT_EQ("//mnt/slow/riak/data/leveldb", options.tiered_slow_prefix);
|
||||
|
||||
// special case with no dbname given, should have no changes
|
||||
fname=MakeTieredDbname("", options);
|
||||
ASSERT_EQ("//mnt/fast/riak/data/leveldb", fname);
|
||||
ASSERT_EQ("//mnt/fast/riak/data/leveldb", options.tiered_fast_prefix);
|
||||
ASSERT_EQ("//mnt/slow/riak/data/leveldb", options.tiered_slow_prefix);
|
||||
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
#include "leveldb/dumpfile.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace leveldb {
|
||||
namespace {
|
||||
|
||||
class StdoutPrinter : public WritableFile {
|
||||
public:
|
||||
virtual Status Append(const Slice& data) {
|
||||
fwrite(data.data(), 1, data.size(), stdout);
|
||||
return Status::OK();
|
||||
}
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
virtual std::string GetName() const { return "[stdout]"; }
|
||||
};
|
||||
|
||||
bool HandleDumpCommand(Env* env, char** files, int num) {
|
||||
StdoutPrinter printer;
|
||||
bool ok = true;
|
||||
for (int i = 0; i < num; i++) {
|
||||
Status s = DumpFile(env, files[i], &printer);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace leveldb
|
||||
|
||||
static void Usage() {
|
||||
fprintf(
|
||||
stderr,
|
||||
"Usage: leveldbutil command...\n"
|
||||
" dump files... -- dump contents of specified files\n"
|
||||
);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
leveldb::Env* env = leveldb::Env::Default();
|
||||
bool ok = true;
|
||||
if (argc < 2) {
|
||||
Usage();
|
||||
ok = false;
|
||||
} else {
|
||||
std::string command = argv[1];
|
||||
if (command == "dump") {
|
||||
ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
|
||||
} else {
|
||||
Usage();
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
return (ok ? 0 : 1);
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// Log format information shared by reader and writer.
|
||||
// See ../doc/log_format.txt for more detail.
|
||||
// See ../doc/log_format.md for more detail.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
@@ -26,8 +26,8 @@ static const int kMaxRecordType = kLastType;
|
||||
|
||||
static const int kBlockSize = 32768;
|
||||
|
||||
// Header is checksum (4 bytes), type (1 byte), length (2 bytes).
|
||||
static const int kHeaderSize = 4 + 1 + 2;
|
||||
// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
|
||||
static const int kHeaderSize = 4 + 2 + 1;
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
|
||||
@@ -25,7 +25,8 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
|
||||
eof_(false),
|
||||
last_record_offset_(0),
|
||||
end_of_buffer_offset_(0),
|
||||
initial_offset_(initial_offset) {
|
||||
initial_offset_(initial_offset),
|
||||
resyncing_(initial_offset > 0) {
|
||||
}
|
||||
|
||||
Reader::~Reader() {
|
||||
@@ -72,8 +73,25 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
|
||||
|
||||
Slice fragment;
|
||||
while (true) {
|
||||
uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
|
||||
const unsigned int record_type = ReadPhysicalRecord(&fragment);
|
||||
|
||||
// ReadPhysicalRecord may have only had an empty trailer remaining in its
|
||||
// internal buffer. Calculate the offset of the next physical record now
|
||||
// that it has returned, properly accounting for its header size.
|
||||
uint64_t physical_record_offset =
|
||||
end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();
|
||||
|
||||
if (resyncing_) {
|
||||
if (record_type == kMiddleType) {
|
||||
continue;
|
||||
} else if (record_type == kLastType) {
|
||||
resyncing_ = false;
|
||||
continue;
|
||||
} else {
|
||||
resyncing_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
switch (record_type) {
|
||||
case kFullType:
|
||||
if (in_fragmented_record) {
|
||||
@@ -133,7 +151,9 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
|
||||
|
||||
case kEof:
|
||||
if (in_fragmented_record) {
|
||||
ReportCorruption(scratch->size(), "partial record without end(3)");
|
||||
// This can be caused by the writer dying immediately after
|
||||
// writing a physical record but before completing the next; don't
|
||||
// treat it as a corruption, just ignore the entire logical record.
|
||||
scratch->clear();
|
||||
}
|
||||
return false;
|
||||
@@ -165,20 +185,20 @@ uint64_t Reader::LastRecordOffset() {
|
||||
return last_record_offset_;
|
||||
}
|
||||
|
||||
void Reader::ReportCorruption(size_t bytes, const char* reason) {
|
||||
ReportDrop(bytes, Status::Corruption(reason));
|
||||
void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
|
||||
ReportDrop(bytes, Status::Corruption(reason, file_->GetName()));
|
||||
}
|
||||
|
||||
void Reader::ReportDrop(size_t bytes, const Status& reason) {
|
||||
void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
|
||||
if (reporter_ != NULL &&
|
||||
end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
|
||||
reporter_->Corruption(bytes, reason);
|
||||
reporter_->Corruption(static_cast<size_t>(bytes), reason);
|
||||
}
|
||||
}
|
||||
|
||||
unsigned int Reader::ReadPhysicalRecord(Slice* result) {
|
||||
while (true) {
|
||||
if (buffer_.size() < (size_t)kHeaderSize) {
|
||||
if (buffer_.size() < kHeaderSize) {
|
||||
if (!eof_) {
|
||||
// Last read was a full read, so this is a trailer to skip
|
||||
buffer_.clear();
|
||||
@@ -189,17 +209,16 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
|
||||
ReportDrop(kBlockSize, status);
|
||||
eof_ = true;
|
||||
return kEof;
|
||||
} else if (buffer_.size() < (size_t)kBlockSize) {
|
||||
} else if (buffer_.size() < kBlockSize) {
|
||||
eof_ = true;
|
||||
}
|
||||
continue;
|
||||
} else if (buffer_.size() == 0) {
|
||||
// End of file
|
||||
return kEof;
|
||||
} else {
|
||||
size_t drop_size = buffer_.size();
|
||||
// Note that if buffer_ is non-empty, we have a truncated header at the
|
||||
// end of the file, which can be caused by the writer crashing in the
|
||||
// middle of writing the header. Instead of considering this an error,
|
||||
// just report EOF.
|
||||
buffer_.clear();
|
||||
ReportCorruption(drop_size, "truncated record at end of file");
|
||||
return kEof;
|
||||
}
|
||||
}
|
||||
@@ -213,8 +232,14 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
|
||||
if (kHeaderSize + length > buffer_.size()) {
|
||||
size_t drop_size = buffer_.size();
|
||||
buffer_.clear();
|
||||
ReportCorruption(drop_size, "bad record length");
|
||||
return kBadRecord;
|
||||
if (!eof_) {
|
||||
ReportCorruption(drop_size, "bad record length");
|
||||
return kBadRecord;
|
||||
}
|
||||
// If the end of the file has been reached without reading |length| bytes
|
||||
// of payload, assume the writer died in the middle of writing the record.
|
||||
// Don't report a corruption.
|
||||
return kEof;
|
||||
}
|
||||
|
||||
if (type == kZeroType && length == 0) {
|
||||
|
||||
@@ -73,6 +73,11 @@ class Reader {
|
||||
// Offset at which to start looking for the first record to return
|
||||
uint64_t const initial_offset_;
|
||||
|
||||
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
|
||||
// particular, a run of kMiddleType and kLastType records can be silently
|
||||
// skipped in this mode
|
||||
bool resyncing_;
|
||||
|
||||
// Extend record types with the following special values
|
||||
enum {
|
||||
kEof = kMaxRecordType + 1,
|
||||
@@ -94,8 +99,8 @@ class Reader {
|
||||
|
||||
// Reports dropped bytes to the reporter.
|
||||
// buffer_ must be updated to remove the dropped bytes prior to invocation.
|
||||
void ReportCorruption(size_t bytes, const char* reason);
|
||||
void ReportDrop(size_t bytes, const Status& reason);
|
||||
void ReportCorruption(uint64_t bytes, const char* reason);
|
||||
void ReportDrop(uint64_t bytes, const Status& reason);
|
||||
|
||||
// No copying allowed
|
||||
Reader(const Reader&);
|
||||
|
||||
+117
-26
@@ -79,7 +79,7 @@ class LogTest {
|
||||
virtual Status Skip(uint64_t n) {
|
||||
if (n > contents_.size()) {
|
||||
contents_.clear();
|
||||
return Status::NotFound("in-memory file skipepd past end");
|
||||
return Status::NotFound("in-memory file skipped past end");
|
||||
}
|
||||
|
||||
contents_.remove_prefix(n);
|
||||
@@ -104,23 +104,34 @@ class LogTest {
|
||||
StringSource source_;
|
||||
ReportCollector report_;
|
||||
bool reading_;
|
||||
Writer writer_;
|
||||
Reader reader_;
|
||||
Writer* writer_;
|
||||
Reader* reader_;
|
||||
|
||||
// Record metadata for testing initial offset functionality
|
||||
static size_t initial_offset_record_sizes_[];
|
||||
static uint64_t initial_offset_last_record_offsets_[];
|
||||
static int num_initial_offset_records_;
|
||||
|
||||
public:
|
||||
LogTest() : reading_(false),
|
||||
writer_(&dest_),
|
||||
reader_(&source_, &report_, true/*checksum*/,
|
||||
0/*initial_offset*/) {
|
||||
writer_(new Writer(&dest_)),
|
||||
reader_(new Reader(&source_, &report_, true/*checksum*/,
|
||||
0/*initial_offset*/)) {
|
||||
}
|
||||
|
||||
~LogTest() {
|
||||
delete writer_;
|
||||
delete reader_;
|
||||
}
|
||||
|
||||
void ReopenForAppend() {
|
||||
delete writer_;
|
||||
writer_ = new Writer(&dest_, dest_.contents_.size());
|
||||
}
|
||||
|
||||
void Write(const std::string& msg) {
|
||||
ASSERT_TRUE(!reading_) << "Write() after starting to read";
|
||||
writer_.AddRecord(Slice(msg));
|
||||
writer_->AddRecord(Slice(msg));
|
||||
}
|
||||
|
||||
size_t WrittenBytes() const {
|
||||
@@ -134,7 +145,7 @@ class LogTest {
|
||||
}
|
||||
std::string scratch;
|
||||
Slice record;
|
||||
if (reader_.ReadRecord(&record, &scratch)) {
|
||||
if (reader_->ReadRecord(&record, &scratch)) {
|
||||
return record.ToString();
|
||||
} else {
|
||||
return "EOF";
|
||||
@@ -182,13 +193,18 @@ class LogTest {
|
||||
}
|
||||
|
||||
void WriteInitialOffsetLog() {
|
||||
for (int i = 0; i < 4; i++) {
|
||||
for (int i = 0; i < num_initial_offset_records_; i++) {
|
||||
std::string record(initial_offset_record_sizes_[i],
|
||||
static_cast<char>('a' + i));
|
||||
Write(record);
|
||||
}
|
||||
}
|
||||
|
||||
void StartReadingAt(uint64_t initial_offset) {
|
||||
delete reader_;
|
||||
reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset);
|
||||
}
|
||||
|
||||
void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
|
||||
WriteInitialOffsetLog();
|
||||
reading_ = true;
|
||||
@@ -208,32 +224,48 @@ class LogTest {
|
||||
source_.contents_ = Slice(dest_.contents_);
|
||||
Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
|
||||
initial_offset);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
|
||||
ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
|
||||
record.size());
|
||||
ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
|
||||
offset_reader->LastRecordOffset());
|
||||
ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
|
||||
|
||||
// Read all records from expected_record_offset through the last one.
|
||||
ASSERT_LT(expected_record_offset, num_initial_offset_records_);
|
||||
for (; expected_record_offset < num_initial_offset_records_;
|
||||
++expected_record_offset) {
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
|
||||
ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
|
||||
record.size());
|
||||
ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
|
||||
offset_reader->LastRecordOffset());
|
||||
ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
|
||||
}
|
||||
delete offset_reader;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
size_t LogTest::initial_offset_record_sizes_[] =
|
||||
{10000, // Two sizable records in first block
|
||||
10000,
|
||||
2 * log::kBlockSize - 1000, // Span three blocks
|
||||
1};
|
||||
1,
|
||||
13716, // Consume all but two bytes of block 3.
|
||||
log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
|
||||
};
|
||||
|
||||
uint64_t LogTest::initial_offset_last_record_offsets_[] =
|
||||
{0,
|
||||
kHeaderSize + 10000,
|
||||
2 * (kHeaderSize + 10000),
|
||||
2 * (kHeaderSize + 10000) +
|
||||
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize};
|
||||
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
|
||||
2 * (kHeaderSize + 10000) +
|
||||
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize
|
||||
+ kHeaderSize + 1,
|
||||
3 * log::kBlockSize,
|
||||
};
|
||||
|
||||
// LogTest::initial_offset_last_record_offsets_ must be defined before this.
|
||||
int LogTest::num_initial_offset_records_ =
|
||||
sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t);
|
||||
|
||||
TEST(LogTest, Empty) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
@@ -318,6 +350,15 @@ TEST(LogTest, AlignedEof) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST(LogTest, OpenForAppend) {
|
||||
Write("hello");
|
||||
ReopenForAppend();
|
||||
Write("world");
|
||||
ASSERT_EQ("hello", Read());
|
||||
ASSERT_EQ("world", Read());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST(LogTest, RandomRead) {
|
||||
const int N = 500;
|
||||
Random write_rnd(301);
|
||||
@@ -351,20 +392,32 @@ TEST(LogTest, BadRecordType) {
|
||||
ASSERT_EQ("OK", MatchError("unknown record type"));
|
||||
}
|
||||
|
||||
TEST(LogTest, TruncatedTrailingRecord) {
|
||||
TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
|
||||
Write("foo");
|
||||
ShrinkSize(4); // Drop all payload as well as a header byte
|
||||
ASSERT_EQ("EOF", Read());
|
||||
ASSERT_EQ(kHeaderSize - 1, DroppedBytes());
|
||||
ASSERT_EQ("OK", MatchError("truncated record at end of file"));
|
||||
// Truncated last record is ignored, not treated as an error.
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
}
|
||||
|
||||
TEST(LogTest, BadLength) {
|
||||
const int kPayloadSize = kBlockSize - kHeaderSize;
|
||||
Write(BigString("bar", kPayloadSize));
|
||||
Write("foo");
|
||||
// Least significant size byte is stored in header[4].
|
||||
IncrementByte(4, 1);
|
||||
ASSERT_EQ("foo", Read());
|
||||
ASSERT_EQ(kBlockSize, DroppedBytes());
|
||||
ASSERT_EQ("OK", MatchError("bad record length"));
|
||||
}
|
||||
|
||||
TEST(LogTest, BadLengthAtEndIsIgnored) {
|
||||
Write("foo");
|
||||
ShrinkSize(1);
|
||||
ASSERT_EQ("EOF", Read());
|
||||
ASSERT_EQ(kHeaderSize + 2, DroppedBytes());
|
||||
ASSERT_EQ("OK", MatchError("bad record length"));
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
}
|
||||
|
||||
TEST(LogTest, ChecksumMismatch) {
|
||||
@@ -415,6 +468,40 @@ TEST(LogTest, UnexpectedFirstType) {
|
||||
ASSERT_EQ("OK", MatchError("partial record without end"));
|
||||
}
|
||||
|
||||
TEST(LogTest, MissingLastIsIgnored) {
|
||||
Write(BigString("bar", kBlockSize));
|
||||
// Remove the LAST block, including header.
|
||||
ShrinkSize(14);
|
||||
ASSERT_EQ("EOF", Read());
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
}
|
||||
|
||||
TEST(LogTest, PartialLastIsIgnored) {
|
||||
Write(BigString("bar", kBlockSize));
|
||||
// Cause a bad record length in the LAST block.
|
||||
ShrinkSize(1);
|
||||
ASSERT_EQ("EOF", Read());
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
}
|
||||
|
||||
TEST(LogTest, SkipIntoMultiRecord) {
|
||||
// Consider a fragmented record:
|
||||
// first(R1), middle(R1), last(R1), first(R2)
|
||||
// If initial_offset points to a record after first(R1) but before first(R2)
|
||||
// incomplete fragment errors are not actual errors, and must be suppressed
|
||||
// until a new first or full record is encountered.
|
||||
Write(BigString("foo", 3*kBlockSize));
|
||||
Write("correct");
|
||||
StartReadingAt(kBlockSize);
|
||||
|
||||
ASSERT_EQ("correct", Read());
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST(LogTest, ErrorJoinsRecords) {
|
||||
// Consider two fragmented records:
|
||||
// first(R1) last(R1) first(R2) last(R2)
|
||||
@@ -433,7 +520,7 @@ TEST(LogTest, ErrorJoinsRecords) {
|
||||
|
||||
ASSERT_EQ("correct", Read());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
const int dropped = DroppedBytes();
|
||||
const size_t dropped = DroppedBytes();
|
||||
ASSERT_LE(dropped, 2*kBlockSize + 100);
|
||||
ASSERT_GE(dropped, 2*kBlockSize);
|
||||
}
|
||||
@@ -484,6 +571,10 @@ TEST(LogTest, ReadFourthStart) {
|
||||
3);
|
||||
}
|
||||
|
||||
TEST(LogTest, ReadInitialOffsetIntoBlockPadding) {
|
||||
CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
|
||||
}
|
||||
|
||||
TEST(LogTest, ReadEnd) {
|
||||
CheckOffsetPastEndReturnsNoRecords(0);
|
||||
}
|
||||
|
||||
@@ -12,13 +12,22 @@
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
static void InitTypeCrc(uint32_t* type_crc) {
|
||||
for (int i = 0; i <= kMaxRecordType; i++) {
|
||||
char t = static_cast<char>(i);
|
||||
type_crc[i] = crc32c::Value(&t, 1);
|
||||
}
|
||||
}
|
||||
|
||||
Writer::Writer(WritableFile* dest)
|
||||
: dest_(dest),
|
||||
block_offset_(0) {
|
||||
for (int i = 0; i <= kMaxRecordType; i++) {
|
||||
char t = static_cast<char>(i);
|
||||
type_crc_[i] = crc32c::Value(&t, 1);
|
||||
}
|
||||
InitTypeCrc(type_crc_);
|
||||
}
|
||||
|
||||
Writer::Writer(WritableFile* dest, uint64_t dest_length)
|
||||
: dest_(dest), block_offset_(dest_length % kBlockSize) {
|
||||
InitTypeCrc(type_crc_);
|
||||
}
|
||||
|
||||
Writer::~Writer() {
|
||||
@@ -74,7 +83,7 @@ Status Writer::AddRecord(const Slice& slice) {
|
||||
|
||||
Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
|
||||
assert(n <= 0xffff); // Must fit in two bytes
|
||||
assert(block_offset_ + kHeaderSize + (int)n <= kBlockSize);
|
||||
assert(block_offset_ + kHeaderSize + n <= kBlockSize);
|
||||
|
||||
// Format the header
|
||||
char buf[kHeaderSize];
|
||||
|
||||
@@ -9,10 +9,11 @@
|
||||
#include "db/log_format.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/env.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class WritableFile;
|
||||
|
||||
namespace log {
|
||||
|
||||
class Writer {
|
||||
@@ -21,12 +22,16 @@ class Writer {
|
||||
// "*dest" must be initially empty.
|
||||
// "*dest" must remain live while this Writer is in use.
|
||||
explicit Writer(WritableFile* dest);
|
||||
|
||||
// Create a writer that will append data to "*dest".
|
||||
// "*dest" must have initial length "dest_length".
|
||||
// "*dest" must remain live while this Writer is in use.
|
||||
Writer(WritableFile* dest, uint64_t dest_length);
|
||||
|
||||
~Writer();
|
||||
|
||||
Status AddRecord(const Slice& slice);
|
||||
|
||||
void Close() {delete dest_; dest_=NULL;};
|
||||
|
||||
private:
|
||||
WritableFile* dest_;
|
||||
int block_offset_; // Current offset in block
|
||||
|
||||
+12
-66
@@ -6,7 +6,6 @@
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/expiry.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
@@ -64,8 +63,6 @@ class MemTableIterator: public Iterator {
|
||||
Slice key_slice = GetLengthPrefixedSlice(iter_.key());
|
||||
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
|
||||
}
|
||||
virtual KeyMetaData & keymetadata() const
|
||||
{MemTable::DecodeKeyMetaData(iter_.key(), keymetadata_); return(keymetadata_);};
|
||||
|
||||
virtual Status status() const { return Status::OK(); }
|
||||
|
||||
@@ -84,8 +81,7 @@ Iterator* MemTable::NewIterator() {
|
||||
|
||||
void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
const Slice& key,
|
||||
const Slice& value,
|
||||
const ExpiryTimeMicros & expiry) {
|
||||
const Slice& value) {
|
||||
// Format of an entry is concatenation of:
|
||||
// key_size : varint32 of internal_key.size()
|
||||
// key bytes : char[internal_key.size()]
|
||||
@@ -93,7 +89,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
// value bytes : char[value.size()]
|
||||
size_t key_size = key.size();
|
||||
size_t val_size = value.size();
|
||||
size_t internal_key_size = key_size + KeySuffixSize(type);
|
||||
size_t internal_key_size = key_size + 8;
|
||||
const size_t encoded_len =
|
||||
VarintLength(internal_key_size) + internal_key_size +
|
||||
VarintLength(val_size) + val_size;
|
||||
@@ -101,22 +97,15 @@ void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
char* p = EncodeVarint32(buf, internal_key_size);
|
||||
memcpy(p, key.data(), key_size);
|
||||
p += key_size;
|
||||
if (IsExpiryKey(type))
|
||||
{
|
||||
EncodeFixed64(p, expiry);
|
||||
p+=8;
|
||||
}
|
||||
EncodeFixed64(p, (s << 8) | type);
|
||||
p += 8;
|
||||
p = EncodeVarint32(p, val_size);
|
||||
memcpy(p, value.data(), val_size);
|
||||
assert((size_t)((p + val_size) - buf) == encoded_len);
|
||||
assert(p + val_size == buf + encoded_len);
|
||||
table_.Insert(buf);
|
||||
}
|
||||
|
||||
bool MemTable::Get(const LookupKey& key, Value* value, Status* s,
|
||||
const Options * options) {
|
||||
bool ret_flag(false);
|
||||
bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
|
||||
Slice memkey = key.memtable_key();
|
||||
Table::Iterator iter(&table_);
|
||||
iter.Seek(memkey.data());
|
||||
@@ -124,7 +113,6 @@ bool MemTable::Get(const LookupKey& key, Value* value, Status* s,
|
||||
// entry format is:
|
||||
// klength varint32
|
||||
// userkey char[klength]
|
||||
// optional uint64
|
||||
// tag uint64
|
||||
// vlength varint32
|
||||
// value char[vlength]
|
||||
@@ -134,66 +122,24 @@ bool MemTable::Get(const LookupKey& key, Value* value, Status* s,
|
||||
const char* entry = iter.key();
|
||||
uint32_t key_length;
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
|
||||
Slice internal_key(key_ptr, key_length);
|
||||
if (comparator_.comparator.user_comparator()->Compare(
|
||||
ExtractUserKey(internal_key),
|
||||
Slice(key_ptr, key_length - 8),
|
||||
key.user_key()) == 0) {
|
||||
// Correct user key
|
||||
KeyMetaData meta;
|
||||
DecodeKeyMetaData(entry, meta);
|
||||
|
||||
switch (meta.m_Type) {
|
||||
case kTypeValueWriteTime:
|
||||
case kTypeValueExplicitExpiry:
|
||||
{
|
||||
bool expired=false;
|
||||
if (NULL!=options && options->ExpiryActivated())
|
||||
expired=options->expiry_module->MemTableCallback(internal_key);
|
||||
if (expired)
|
||||
{
|
||||
// like kTypeDeletion
|
||||
*s = Status::NotFound(Slice());
|
||||
ret_flag=true;
|
||||
break;
|
||||
} // if
|
||||
//otherwise fall into kTypeValue code
|
||||
} // case
|
||||
|
||||
case kTypeValue:
|
||||
{
|
||||
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
|
||||
switch (static_cast<ValueType>(tag & 0xff)) {
|
||||
case kTypeValue: {
|
||||
Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
|
||||
value->assign(v.data(), v.size());
|
||||
ret_flag=true;
|
||||
break;
|
||||
return true;
|
||||
}
|
||||
case kTypeDeletion:
|
||||
*s = Status::NotFound(Slice());
|
||||
ret_flag=true;
|
||||
break;
|
||||
} // switch
|
||||
|
||||
// only unpack metadata if requested
|
||||
if (key.WantsKeyMetaData())
|
||||
key.SetKeyMetaData(meta);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret_flag;
|
||||
return false;
|
||||
}
|
||||
|
||||
// this is a static function
|
||||
void MemTable::DecodeKeyMetaData(
|
||||
const char * key,
|
||||
KeyMetaData & meta)
|
||||
{
|
||||
Slice key_slice = GetLengthPrefixedSlice(key);
|
||||
|
||||
meta.m_Type=ExtractValueType(key_slice);
|
||||
meta.m_Sequence=ExtractSequenceNumber(key_slice);
|
||||
if (IsExpiryKey(meta.m_Type))
|
||||
meta.m_Expiry=ExtractExpiry(key_slice);
|
||||
else
|
||||
meta.m_Expiry=0;
|
||||
|
||||
} // DecodeKeyMetaData
|
||||
|
||||
} // namespace leveldb
|
||||
@@ -24,10 +24,10 @@ class MemTable {
|
||||
explicit MemTable(const InternalKeyComparator& comparator);
|
||||
|
||||
// Increase reference count.
|
||||
void Ref() volatile { ++refs_; }
|
||||
void Ref() { ++refs_; }
|
||||
|
||||
// Drop reference count. Delete if no more references exist.
|
||||
void Unref() volatile {
|
||||
void Unref() {
|
||||
--refs_;
|
||||
assert(refs_ >= 0);
|
||||
if (refs_ <= 0) {
|
||||
@@ -36,10 +36,7 @@ class MemTable {
|
||||
}
|
||||
|
||||
// Returns an estimate of the number of bytes of data in use by this
|
||||
// data structure.
|
||||
//
|
||||
// REQUIRES: external synchronization to prevent simultaneous
|
||||
// operations on the same MemTable.
|
||||
// data structure. It is safe to call when MemTable is being modified.
|
||||
size_t ApproximateMemoryUsage();
|
||||
|
||||
// Return an iterator that yields the contents of the memtable.
|
||||
@@ -55,17 +52,13 @@ class MemTable {
|
||||
// Typically value will be empty if type==kTypeDeletion.
|
||||
void Add(SequenceNumber seq, ValueType type,
|
||||
const Slice& key,
|
||||
const Slice& value,
|
||||
const ExpiryTimeMicros& expiry=0);
|
||||
const Slice& value);
|
||||
|
||||
// If memtable contains a value for key, store it in *value and return true.
|
||||
// If memtable contains a deletion for key, store a NotFound() error
|
||||
// in *status and return true.
|
||||
// Else, return false.
|
||||
bool Get(const LookupKey& key, Value* value, Status* s, const Options * options);
|
||||
|
||||
// parse keymetadata from skiplist key string
|
||||
static void DecodeKeyMetaData(const char * key, KeyMetaData & meta);
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s);
|
||||
|
||||
private:
|
||||
~MemTable(); // Private since only Unref() should be used to delete it
|
||||
@@ -76,7 +69,7 @@ class MemTable {
|
||||
int operator()(const char* a, const char* b) const;
|
||||
};
|
||||
friend class MemTableIterator;
|
||||
friend class MemTableBackwardIterator; // does not exist
|
||||
friend class MemTableBackwardIterator;
|
||||
|
||||
typedef SkipList<const char*, KeyComparator> Table;
|
||||
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
// -------------------------------------------------------------------
|
||||
//
|
||||
// penalty_test.cc
|
||||
//
|
||||
// Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
//
|
||||
// This file is provided to you under the Apache License,
|
||||
// Version 2.0 (the "License"); you may not use this file
|
||||
// except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
//
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
#include "leveldb/comparator.h"
|
||||
|
||||
#include "db/version_set.h"
|
||||
|
||||
/**
|
||||
* Execution routine
|
||||
*/
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class TestVersion : public Version
|
||||
{
|
||||
public:
|
||||
TestVersion()
|
||||
: Version(NULL)
|
||||
{
|
||||
int loop;
|
||||
|
||||
for (loop=0; loop<config::kNumLevels; ++loop)
|
||||
{
|
||||
m_FalseFile[loop].file_size=0;
|
||||
m_LevelFileCount[loop]=0;
|
||||
} // for
|
||||
};
|
||||
|
||||
virtual size_t NumFiles(int level) const {return(m_LevelFileCount[level]);};
|
||||
|
||||
virtual const std::vector<FileMetaData*> & GetFileList(int level) const
|
||||
{
|
||||
m_FalseVector.clear();
|
||||
m_FalseVector.push_back(&m_FalseFile[level]);
|
||||
return(m_FalseVector);
|
||||
};
|
||||
|
||||
mutable std::vector<FileMetaData*> m_FalseVector;
|
||||
mutable FileMetaData m_FalseFile[config::kNumLevels];
|
||||
|
||||
size_t m_LevelFileCount[config::kNumLevels];
|
||||
|
||||
}; // class TestVersion
|
||||
|
||||
/**
|
||||
* Wrapper class for tests. Holds working variables
|
||||
* and helper functions.
|
||||
*/
|
||||
class PenaltyTester : public VersionSet
|
||||
{
|
||||
public:
|
||||
PenaltyTester()
|
||||
: m_IntCompare(m_Options.comparator), VersionSet("", &m_Options, NULL, &m_IntCompare)
|
||||
{
|
||||
};
|
||||
|
||||
~PenaltyTester()
|
||||
{
|
||||
};
|
||||
|
||||
Options m_Options;
|
||||
InternalKeyComparator m_IntCompare;
|
||||
|
||||
}; // class PenaltyTester
|
||||
|
||||
|
||||
/*******************
|
||||
* Form note:
|
||||
* using ASSERT_TRUE(0==version.WritePenalty());
|
||||
* instead of ASSERT_EQ / ASSERT_NE because WritePenalty
|
||||
* returns a volatile int, which older compilers believe is
|
||||
* not an equivalent type to a constant. RedHat 5, Solaris,
|
||||
* and SmartOS were giving grief.
|
||||
*******************/
|
||||
|
||||
/**
|
||||
* Debug 1
|
||||
*/
|
||||
#if 0
|
||||
TEST(PenaltyTester, Debug1)
|
||||
{
|
||||
TestVersion version;
|
||||
int penalty;
|
||||
|
||||
m_Options.write_buffer_size=46416847;
|
||||
|
||||
version.m_FalseFile[2].file_size=1075676398;
|
||||
version.m_LevelFileCount[1]=1;
|
||||
|
||||
UpdatePenalty(&version);
|
||||
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
} // test Debug1
|
||||
#endif
|
||||
|
||||
|
||||
/**
|
||||
* No penalty scenarios
|
||||
*/
|
||||
TEST(PenaltyTester, NoPenalty)
|
||||
{
|
||||
TestVersion version;
|
||||
int level;
|
||||
|
||||
m_Options.write_buffer_size=46416847;
|
||||
|
||||
// nothing
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
/**
|
||||
* Level 0
|
||||
* (overlapped level, penalty is count based)
|
||||
*/
|
||||
// no penalty
|
||||
version.m_LevelFileCount[0]=config::kL0_CompactionTrigger;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_LevelFileCount[0]=config::kL0_SlowdownWritesTrigger;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
#if 0 // needs rewrite to be time based
|
||||
// threshold reached ... some penalty
|
||||
version.m_LevelFileCount[0]=config::kL0_SlowdownWritesTrigger+1;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0!=version.WritePenalty());
|
||||
|
||||
// clean up
|
||||
version.m_LevelFileCount[0]=0;
|
||||
|
||||
/**
|
||||
* Level 1
|
||||
* (overlapped level, penalty is count based)
|
||||
*/
|
||||
// no penalty
|
||||
version.m_LevelFileCount[1]=config::kL0_CompactionTrigger;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_LevelFileCount[1]=config::kL0_SlowdownWritesTrigger;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
// threshold reached ... some penalty
|
||||
version.m_LevelFileCount[1]=config::kL0_SlowdownWritesTrigger+1;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0!=version.WritePenalty());
|
||||
|
||||
// clean up
|
||||
version.m_LevelFileCount[1]=0;
|
||||
|
||||
/**
|
||||
* Level 2
|
||||
* (landing level, penalty size based)
|
||||
*/
|
||||
// no penalty
|
||||
version.m_FalseFile[2].file_size=0;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_FalseFile[2].file_size=VersionSet::DesiredBytesForLevel(2);
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_FalseFile[2].file_size=VersionSet::MaxBytesForLevel(2)-1;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_FalseFile[2].file_size=VersionSet::MaxBytesForLevel(2);
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0!=version.WritePenalty());
|
||||
|
||||
// interaction rule with level 1
|
||||
version.m_FalseFile[2].file_size=VersionSet::MaxBytesForLevel(2)-1;
|
||||
version.m_LevelFileCount[1]=config::kL0_CompactionTrigger/2;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0!=version.WritePenalty());
|
||||
|
||||
// clean up
|
||||
version.m_LevelFileCount[1]=0;
|
||||
version.m_FalseFile[2].file_size=0;
|
||||
|
||||
/**
|
||||
* Level 3+
|
||||
* (landing level, penalty size based)
|
||||
*/
|
||||
for (level=3; level<config::kNumLevels; ++level)
|
||||
{
|
||||
// no penalty
|
||||
version.m_FalseFile[level].file_size=0;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_FalseFile[level].file_size=VersionSet::DesiredBytesForLevel(level);
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_FalseFile[level].file_size=VersionSet::MaxBytesForLevel(level)-1;
|
||||
UpdatePenalty(&version);
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
version.m_FalseFile[level].file_size=VersionSet::MaxBytesForLevel(level);
|
||||
UpdatePenalty(&version);
|
||||
if ((config::kNumLevels-1)!=level)
|
||||
ASSERT_TRUE(0!=version.WritePenalty());
|
||||
else
|
||||
ASSERT_TRUE(0==version.WritePenalty());
|
||||
|
||||
// clean up
|
||||
version.m_FalseFile[level].file_size=0;
|
||||
} // for
|
||||
#endif
|
||||
} // test NoPenalty
|
||||
|
||||
|
||||
|
||||
} // namespace leveldb
|
||||
@@ -0,0 +1,324 @@
|
||||
// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class RecoveryTest {
|
||||
public:
|
||||
RecoveryTest() : env_(Env::Default()), db_(NULL) {
|
||||
dbname_ = test::TmpDir() + "/recovery_test";
|
||||
DestroyDB(dbname_, Options());
|
||||
Open();
|
||||
}
|
||||
|
||||
~RecoveryTest() {
|
||||
Close();
|
||||
DestroyDB(dbname_, Options());
|
||||
}
|
||||
|
||||
DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
|
||||
Env* env() const { return env_; }
|
||||
|
||||
bool CanAppend() {
|
||||
WritableFile* tmp;
|
||||
Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
|
||||
delete tmp;
|
||||
if (s.IsNotSupportedError()) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
void Close() {
|
||||
delete db_;
|
||||
db_ = NULL;
|
||||
}
|
||||
|
||||
void Open(Options* options = NULL) {
|
||||
Close();
|
||||
Options opts;
|
||||
if (options != NULL) {
|
||||
opts = *options;
|
||||
} else {
|
||||
opts.reuse_logs = true; // TODO(sanjay): test both ways
|
||||
opts.create_if_missing = true;
|
||||
}
|
||||
if (opts.env == NULL) {
|
||||
opts.env = env_;
|
||||
}
|
||||
ASSERT_OK(DB::Open(opts, dbname_, &db_));
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
}
|
||||
|
||||
Status Put(const std::string& k, const std::string& v) {
|
||||
return db_->Put(WriteOptions(), k, v);
|
||||
}
|
||||
|
||||
std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
|
||||
std::string result;
|
||||
Status s = db_->Get(ReadOptions(), k, &result);
|
||||
if (s.IsNotFound()) {
|
||||
result = "NOT_FOUND";
|
||||
} else if (!s.ok()) {
|
||||
result = s.ToString();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string ManifestFileName() {
|
||||
std::string current;
|
||||
ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t));
|
||||
size_t len = current.size();
|
||||
if (len > 0 && current[len-1] == '\n') {
|
||||
current.resize(len - 1);
|
||||
}
|
||||
return dbname_ + "/" + current;
|
||||
}
|
||||
|
||||
std::string LogName(uint64_t number) {
|
||||
return LogFileName(dbname_, number);
|
||||
}
|
||||
|
||||
size_t DeleteLogFiles() {
|
||||
std::vector<uint64_t> logs = GetFiles(kLogFile);
|
||||
for (size_t i = 0; i < logs.size(); i++) {
|
||||
ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
|
||||
}
|
||||
return logs.size();
|
||||
}
|
||||
|
||||
uint64_t FirstLogFile() {
|
||||
return GetFiles(kLogFile)[0];
|
||||
}
|
||||
|
||||
std::vector<uint64_t> GetFiles(FileType t) {
|
||||
std::vector<std::string> filenames;
|
||||
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
|
||||
std::vector<uint64_t> result;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
if (ParseFileName(filenames[i], &number, &type) && type == t) {
|
||||
result.push_back(number);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
int NumLogs() {
|
||||
return GetFiles(kLogFile).size();
|
||||
}
|
||||
|
||||
int NumTables() {
|
||||
return GetFiles(kTableFile).size();
|
||||
}
|
||||
|
||||
uint64_t FileSize(const std::string& fname) {
|
||||
uint64_t result;
|
||||
ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
|
||||
return result;
|
||||
}
|
||||
|
||||
void CompactMemTable() {
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
}
|
||||
|
||||
// Directly construct a log file that sets key to val.
|
||||
void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
|
||||
std::string fname = LogFileName(dbname_, lognum);
|
||||
WritableFile* file;
|
||||
ASSERT_OK(env_->NewWritableFile(fname, &file));
|
||||
log::Writer writer(file);
|
||||
WriteBatch batch;
|
||||
batch.Put(key, val);
|
||||
WriteBatchInternal::SetSequence(&batch, seq);
|
||||
ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
|
||||
ASSERT_OK(file->Flush());
|
||||
delete file;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string dbname_;
|
||||
Env* env_;
|
||||
DB* db_;
|
||||
};
|
||||
|
||||
TEST(RecoveryTest, ManifestReused) {
|
||||
if (!CanAppend()) {
|
||||
fprintf(stderr, "skipping test because env does not support appending\n");
|
||||
return;
|
||||
}
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
std::string old_manifest = ManifestFileName();
|
||||
Open();
|
||||
ASSERT_EQ(old_manifest, ManifestFileName());
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
Open();
|
||||
ASSERT_EQ(old_manifest, ManifestFileName());
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
|
||||
TEST(RecoveryTest, LargeManifestCompacted) {
|
||||
if (!CanAppend()) {
|
||||
fprintf(stderr, "skipping test because env does not support appending\n");
|
||||
return;
|
||||
}
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
std::string old_manifest = ManifestFileName();
|
||||
|
||||
// Pad with zeroes to make manifest file very big.
|
||||
{
|
||||
uint64_t len = FileSize(old_manifest);
|
||||
WritableFile* file;
|
||||
ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
|
||||
std::string zeroes(3*1048576 - static_cast<size_t>(len), 0);
|
||||
ASSERT_OK(file->Append(zeroes));
|
||||
ASSERT_OK(file->Flush());
|
||||
delete file;
|
||||
}
|
||||
|
||||
Open();
|
||||
std::string new_manifest = ManifestFileName();
|
||||
ASSERT_NE(old_manifest, new_manifest);
|
||||
ASSERT_GT(10000, FileSize(new_manifest));
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
|
||||
Open();
|
||||
ASSERT_EQ(new_manifest, ManifestFileName());
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
|
||||
TEST(RecoveryTest, NoLogFiles) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_EQ(1, DeleteLogFiles());
|
||||
Open();
|
||||
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
||||
Open();
|
||||
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
||||
}
|
||||
|
||||
TEST(RecoveryTest, LogFileReuse) {
|
||||
if (!CanAppend()) {
|
||||
fprintf(stderr, "skipping test because env does not support appending\n");
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < 2; i++) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
if (i == 0) {
|
||||
// Compact to ensure current log is empty
|
||||
CompactMemTable();
|
||||
}
|
||||
Close();
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
uint64_t number = FirstLogFile();
|
||||
if (i == 0) {
|
||||
ASSERT_EQ(0, FileSize(LogName(number)));
|
||||
} else {
|
||||
ASSERT_LT(0, FileSize(LogName(number)));
|
||||
}
|
||||
Open();
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
Open();
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(RecoveryTest, MultipleMemTables) {
|
||||
// Make a large log.
|
||||
const int kNum = 1000;
|
||||
for (int i = 0; i < kNum; i++) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "%050d", i);
|
||||
ASSERT_OK(Put(buf, buf));
|
||||
}
|
||||
ASSERT_EQ(0, NumTables());
|
||||
Close();
|
||||
ASSERT_EQ(0, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
uint64_t old_log_file = FirstLogFile();
|
||||
|
||||
// Force creation of multiple memtables by reducing the write buffer size.
|
||||
Options opt;
|
||||
opt.reuse_logs = true;
|
||||
opt.write_buffer_size = (kNum*100) / 2;
|
||||
Open(&opt);
|
||||
ASSERT_LE(2, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
|
||||
for (int i = 0; i < kNum; i++) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "%050d", i);
|
||||
ASSERT_EQ(buf, Get(buf));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(RecoveryTest, MultipleLogFiles) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
|
||||
// Make a bunch of uncompacted log files.
|
||||
uint64_t old_log = FirstLogFile();
|
||||
MakeLogFile(old_log+1, 1000, "hello", "world");
|
||||
MakeLogFile(old_log+2, 1001, "hi", "there");
|
||||
MakeLogFile(old_log+3, 1002, "foo", "bar2");
|
||||
|
||||
// Recover and check that all log files were processed.
|
||||
Open();
|
||||
ASSERT_LE(1, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
uint64_t new_log = FirstLogFile();
|
||||
ASSERT_LE(old_log+3, new_log);
|
||||
ASSERT_EQ("bar2", Get("foo"));
|
||||
ASSERT_EQ("world", Get("hello"));
|
||||
ASSERT_EQ("there", Get("hi"));
|
||||
|
||||
// Test that previous recovery produced recoverable state.
|
||||
Open();
|
||||
ASSERT_LE(1, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
if (CanAppend()) {
|
||||
ASSERT_EQ(new_log, FirstLogFile());
|
||||
}
|
||||
ASSERT_EQ("bar2", Get("foo"));
|
||||
ASSERT_EQ("world", Get("hello"));
|
||||
ASSERT_EQ("there", Get("hi"));
|
||||
|
||||
// Check that introducing an older log file does not cause it to be re-read.
|
||||
Close();
|
||||
MakeLogFile(old_log+1, 2000, "hello", "stale write");
|
||||
Open();
|
||||
ASSERT_LE(1, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
if (CanAppend()) {
|
||||
ASSERT_EQ(new_log, FirstLogFile());
|
||||
}
|
||||
ASSERT_EQ("bar2", Get("foo"));
|
||||
ASSERT_EQ("world", Get("hello"));
|
||||
ASSERT_EQ("there", Get("hi"));
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+184
-248
@@ -45,113 +45,49 @@ namespace {
|
||||
class Repairer {
|
||||
public:
|
||||
Repairer(const std::string& dbname, const Options& options)
|
||||
: double_cache_(options),
|
||||
options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options, double_cache_.GetBlockCache())),
|
||||
org_options_(options),
|
||||
dbname_(options_.tiered_fast_prefix),
|
||||
org_dbname_(dbname),
|
||||
: dbname_(dbname),
|
||||
env_(options.env),
|
||||
icmp_(options.comparator),
|
||||
ipolicy_(options.filter_policy),
|
||||
options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
|
||||
owns_info_log_(options_.info_log != options.info_log),
|
||||
db_lock_(NULL),
|
||||
next_file_number_(1)
|
||||
{
|
||||
owns_cache_(options_.block_cache != options.block_cache),
|
||||
next_file_number_(1) {
|
||||
// TableCache can be small since we expect each table to be opened once.
|
||||
table_cache_ = new TableCache(dbname_, &options_, double_cache_.GetFileCache(), double_cache_);
|
||||
|
||||
table_cache_ = new TableCache(dbname_, &options_, 10);
|
||||
}
|
||||
|
||||
~Repairer() {
|
||||
delete table_cache_;
|
||||
if (owns_info_log_) {
|
||||
delete options_.info_log;
|
||||
}
|
||||
// if (owns_cache_) {
|
||||
// delete options_.block_cache;
|
||||
// }
|
||||
|
||||
// must remove second ref counter that keeps overlapped files locked
|
||||
// table cache
|
||||
bool is_overlap;
|
||||
for (int level = 0; level < config::kNumLevels; level++) {
|
||||
{
|
||||
is_overlap=(level < leveldb::config::kNumOverlapLevels);
|
||||
for (size_t i = 0; i < table_numbers_[level].size(); i++) {
|
||||
table_cache_->Evict(table_numbers_[level][i], is_overlap);
|
||||
} // for
|
||||
} // if
|
||||
} // for
|
||||
|
||||
delete table_cache_;
|
||||
if (owns_cache_) {
|
||||
delete options_.block_cache;
|
||||
}
|
||||
}
|
||||
|
||||
Status Run() {
|
||||
Status status;
|
||||
|
||||
status = env_->LockFile(LockFileName(dbname_), &db_lock_);
|
||||
|
||||
if (status.ok())
|
||||
status = MakeLevelDirectories(env_, options_);
|
||||
|
||||
Status status = FindFiles();
|
||||
if (status.ok()) {
|
||||
status = FindFiles();
|
||||
if (status.ok()) {
|
||||
ConvertLogFilesToTables();
|
||||
ExtractMetaData();
|
||||
status = WriteDescriptor();
|
||||
ConvertLogFilesToTables();
|
||||
ExtractMetaData();
|
||||
status = WriteDescriptor();
|
||||
}
|
||||
if (status.ok()) {
|
||||
unsigned long long bytes = 0;
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
bytes += tables_[i].meta.file_size;
|
||||
}
|
||||
if (status.ok()) {
|
||||
unsigned long long bytes = 0;
|
||||
unsigned long long files = 0;
|
||||
|
||||
// calculate size for log information
|
||||
for (int level=0; level<config::kNumLevels;++level)
|
||||
{
|
||||
std::vector<TableInfo> * table_ptr;
|
||||
std::vector<TableInfo>::const_iterator i;
|
||||
|
||||
table_ptr=&tables_[level];
|
||||
files+=table_ptr->size();
|
||||
|
||||
for ( i = table_ptr->begin(); table_ptr->end()!= i; i++) {
|
||||
bytes += i->meta.file_size;
|
||||
}
|
||||
} // for
|
||||
|
||||
Log(options_.info_log,
|
||||
"**** Repaired leveldb %s; "
|
||||
"recovered %d files; %llu bytes. "
|
||||
"Some data may have been lost. "
|
||||
"****",
|
||||
dbname_.c_str(),
|
||||
static_cast<int>(files),
|
||||
bytes);
|
||||
}
|
||||
if (db_lock_ != NULL) {
|
||||
env_->UnlockFile(db_lock_);
|
||||
}
|
||||
Log(options_.info_log,
|
||||
"**** Repaired leveldb %s; "
|
||||
"recovered %d files; %llu bytes. "
|
||||
"Some data may have been lost. "
|
||||
"****",
|
||||
dbname_.c_str(),
|
||||
static_cast<int>(tables_.size()),
|
||||
bytes);
|
||||
}
|
||||
|
||||
// perform Riak specific scan for overlapping .sst files
|
||||
// within a level
|
||||
if (status.ok())
|
||||
{
|
||||
leveldb::DB * db_ptr;
|
||||
Options options;
|
||||
|
||||
db_ptr=NULL;
|
||||
options=org_options_;
|
||||
// options.block_cache=NULL; // not reusing for fear of edge cases
|
||||
options.is_repair=true;
|
||||
options.error_if_exists=false;
|
||||
status=leveldb::DB::Open(options, org_dbname_, &db_ptr);
|
||||
|
||||
if (status.ok())
|
||||
status=db_ptr->VerifyLevels();
|
||||
|
||||
delete db_ptr;
|
||||
|
||||
} // if
|
||||
return status;
|
||||
}
|
||||
|
||||
@@ -161,36 +97,34 @@ class Repairer {
|
||||
SequenceNumber max_sequence;
|
||||
};
|
||||
|
||||
DoubleCache double_cache_;
|
||||
Options const options_, org_options_;
|
||||
std::string const dbname_, org_dbname_;
|
||||
std::string const dbname_;
|
||||
Env* const env_;
|
||||
InternalKeyComparator const icmp_;
|
||||
InternalFilterPolicy const ipolicy_;
|
||||
Options const options_;
|
||||
bool owns_info_log_;
|
||||
FileLock* db_lock_;
|
||||
bool owns_cache_;
|
||||
TableCache* table_cache_;
|
||||
VersionEdit edit_;
|
||||
|
||||
std::vector<std::string> manifests_;
|
||||
std::vector<uint64_t> table_numbers_[config::kNumLevels];
|
||||
std::vector<uint64_t> table_numbers_;
|
||||
std::vector<uint64_t> logs_;
|
||||
std::vector<TableInfo> tables_[config::kNumLevels];
|
||||
std::vector<TableInfo> tables_;
|
||||
uint64_t next_file_number_;
|
||||
|
||||
Status FindFiles()
|
||||
{
|
||||
Status FindFiles() {
|
||||
std::vector<std::string> filenames;
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
int level;
|
||||
|
||||
// base directory
|
||||
Status status = env_->GetChildren(dbname_, &filenames);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
if (filenames.empty()) {
|
||||
return Status::IOError(dbname_, "repair found no files");
|
||||
}
|
||||
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type)) {
|
||||
if (type == kDescriptorFile) {
|
||||
@@ -202,38 +136,13 @@ class Repairer {
|
||||
if (type == kLogFile) {
|
||||
logs_.push_back(number);
|
||||
} else if (type == kTableFile) {
|
||||
table_numbers_[0].push_back(number);
|
||||
table_numbers_.push_back(number);
|
||||
} else {
|
||||
// Ignore other files
|
||||
} // else
|
||||
} // else
|
||||
} // if
|
||||
} // for
|
||||
|
||||
for (level=0; level < config::kNumLevels; ++level)
|
||||
{
|
||||
std::string dirname;
|
||||
|
||||
filenames.clear();
|
||||
dirname=MakeDirName2(options_, level, "sst");
|
||||
Status status = env_->GetChildren(dirname, &filenames);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type)) {
|
||||
if (number + 1 > next_file_number_) {
|
||||
next_file_number_ = number + 1;
|
||||
}
|
||||
|
||||
if (type == kTableFile) {
|
||||
table_numbers_[level].push_back(number);
|
||||
}
|
||||
} // if
|
||||
} // for
|
||||
} // for
|
||||
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
@@ -277,7 +186,7 @@ class Repairer {
|
||||
reporter.env = env_;
|
||||
reporter.info_log = options_.info_log;
|
||||
reporter.lognum = log;
|
||||
// We intentially make log::Reader do checksumming so that
|
||||
// We intentionally make log::Reader do checksumming so that
|
||||
// corruptions cause entire commits to be skipped instead of
|
||||
// propagating bad information (like overly large sequence
|
||||
// numbers).
|
||||
@@ -294,11 +203,11 @@ class Repairer {
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(
|
||||
record.size(), Status::Corruption("log record too small"));
|
||||
record.size(), Status::Corruption("log record too small", logname));
|
||||
continue;
|
||||
}
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
status = WriteBatchInternal::InsertInto(&batch, mem, &options_);
|
||||
status = WriteBatchInternal::InsertInto(&batch, mem);
|
||||
if (status.ok()) {
|
||||
counter += WriteBatchInternal::Count(&batch);
|
||||
} else {
|
||||
@@ -314,15 +223,14 @@ class Repairer {
|
||||
// since ExtractMetaData() will also generate edits.
|
||||
FileMetaData meta;
|
||||
meta.number = next_file_number_++;
|
||||
meta.level = 0;
|
||||
Iterator* iter = mem->NewIterator();
|
||||
status = BuildTable(dbname_, env_, options_, icmp_.user_comparator(), table_cache_, iter, &meta, 0);
|
||||
status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
|
||||
delete iter;
|
||||
mem->Unref();
|
||||
mem = NULL;
|
||||
if (status.ok()) {
|
||||
if (meta.file_size > 0) {
|
||||
table_numbers_[0].push_back(meta.number);
|
||||
table_numbers_.push_back(meta.number);
|
||||
}
|
||||
}
|
||||
Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
|
||||
@@ -334,128 +242,168 @@ class Repairer {
|
||||
}
|
||||
|
||||
void ExtractMetaData() {
|
||||
for (int level=0; level < config::kNumLevels; ++level)
|
||||
{
|
||||
std::vector<uint64_t> * number_ptr;
|
||||
std::vector<uint64_t>::const_iterator i;
|
||||
|
||||
number_ptr=&table_numbers_[level];
|
||||
for (i = number_ptr->begin(); number_ptr->end()!= i; ++i) {
|
||||
TableInfo t;
|
||||
t.meta.number = *i;
|
||||
t.meta.level = level;
|
||||
Status status = ScanTable(&t);
|
||||
if (!status.ok())
|
||||
{
|
||||
std::string fname = TableFileName(options_, t.meta.number, t.meta.level);
|
||||
Log(options_.info_log, "Table #%llu: ignoring %s",
|
||||
(unsigned long long) t.meta.number,
|
||||
status.ToString().c_str());
|
||||
ArchiveFile(fname, true);
|
||||
} else {
|
||||
tables_[level].push_back(t);
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < table_numbers_.size(); i++) {
|
||||
ScanTable(table_numbers_[i]);
|
||||
}
|
||||
}
|
||||
|
||||
Status ScanTable(TableInfo* t) {
|
||||
Table * table_ptr;
|
||||
SstCounters counters;
|
||||
std::string fname = TableFileName(options_, t->meta.number, t->meta.level);
|
||||
int counter = 0;
|
||||
Status status = env_->GetFileSize(fname, &t->meta.file_size);
|
||||
if (status.ok()) {
|
||||
Iterator* iter = table_cache_->NewIterator(
|
||||
ReadOptions(), t->meta.number, t->meta.file_size, t->meta.level, &table_ptr);
|
||||
bool empty = true;
|
||||
ParsedInternalKey parsed;
|
||||
t->max_sequence = 0;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
Slice key = iter->key();
|
||||
if (!ParseInternalKey(key, &parsed)) {
|
||||
Log(options_.info_log, "Table #%llu: unparsable key %s",
|
||||
(unsigned long long) t->meta.number,
|
||||
EscapeString(key).c_str());
|
||||
continue;
|
||||
}
|
||||
Iterator* NewTableIterator(const FileMetaData& meta) {
|
||||
// Same as compaction iterators: if paranoid_checks are on, turn
|
||||
// on checksum verification.
|
||||
ReadOptions r;
|
||||
r.verify_checksums = options_.paranoid_checks;
|
||||
return table_cache_->NewIterator(r, meta.number, meta.file_size);
|
||||
}
|
||||
|
||||
counter++;
|
||||
if (empty) {
|
||||
empty = false;
|
||||
t->meta.smallest.DecodeFrom(key);
|
||||
}
|
||||
t->meta.largest.DecodeFrom(key);
|
||||
if (parsed.sequence > t->max_sequence) {
|
||||
t->max_sequence = parsed.sequence;
|
||||
}
|
||||
void ScanTable(uint64_t number) {
|
||||
TableInfo t;
|
||||
t.meta.number = number;
|
||||
std::string fname = TableFileName(dbname_, number);
|
||||
Status status = env_->GetFileSize(fname, &t.meta.file_size);
|
||||
if (!status.ok()) {
|
||||
// Try alternate file name.
|
||||
fname = SSTTableFileName(dbname_, number);
|
||||
Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
|
||||
if (s2.ok()) {
|
||||
status = Status::OK();
|
||||
}
|
||||
if (!iter->status().ok()) {
|
||||
status = iter->status();
|
||||
}
|
||||
else {
|
||||
counters=table_ptr->GetSstCounters();
|
||||
t->meta.exp_write_low=counters.Value(eSstCountExpiry1);
|
||||
t->meta.exp_write_high=counters.Value(eSstCountExpiry2);
|
||||
t->meta.exp_explicit_high=counters.Value(eSstCountExpiry3);
|
||||
}
|
||||
delete iter;
|
||||
}
|
||||
if (!status.ok()) {
|
||||
ArchiveFile(TableFileName(dbname_, number));
|
||||
ArchiveFile(SSTTableFileName(dbname_, number));
|
||||
Log(options_.info_log, "Table #%llu: dropped: %s",
|
||||
(unsigned long long) t.meta.number,
|
||||
status.ToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract metadata by scanning through table.
|
||||
int counter = 0;
|
||||
Iterator* iter = NewTableIterator(t.meta);
|
||||
bool empty = true;
|
||||
ParsedInternalKey parsed;
|
||||
t.max_sequence = 0;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
Slice key = iter->key();
|
||||
if (!ParseInternalKey(key, &parsed)) {
|
||||
Log(options_.info_log, "Table #%llu: unparsable key %s",
|
||||
(unsigned long long) t.meta.number,
|
||||
EscapeString(key).c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
counter++;
|
||||
if (empty) {
|
||||
empty = false;
|
||||
t.meta.smallest.DecodeFrom(key);
|
||||
}
|
||||
t.meta.largest.DecodeFrom(key);
|
||||
if (parsed.sequence > t.max_sequence) {
|
||||
t.max_sequence = parsed.sequence;
|
||||
}
|
||||
}
|
||||
if (!iter->status().ok()) {
|
||||
status = iter->status();
|
||||
}
|
||||
delete iter;
|
||||
Log(options_.info_log, "Table #%llu: %d entries %s",
|
||||
(unsigned long long) t->meta.number,
|
||||
(unsigned long long) t.meta.number,
|
||||
counter,
|
||||
status.ToString().c_str());
|
||||
return status;
|
||||
|
||||
if (status.ok()) {
|
||||
tables_.push_back(t);
|
||||
} else {
|
||||
RepairTable(fname, t); // RepairTable archives input file.
|
||||
}
|
||||
}
|
||||
|
||||
void RepairTable(const std::string& src, TableInfo t) {
|
||||
// We will copy src contents to a new table and then rename the
|
||||
// new table over the source.
|
||||
|
||||
// Create builder.
|
||||
std::string copy = TableFileName(dbname_, next_file_number_++);
|
||||
WritableFile* file;
|
||||
Status s = env_->NewWritableFile(copy, &file);
|
||||
if (!s.ok()) {
|
||||
return;
|
||||
}
|
||||
TableBuilder* builder = new TableBuilder(options_, file);
|
||||
|
||||
// Copy data.
|
||||
Iterator* iter = NewTableIterator(t.meta);
|
||||
int counter = 0;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
builder->Add(iter->key(), iter->value());
|
||||
counter++;
|
||||
}
|
||||
delete iter;
|
||||
|
||||
ArchiveFile(src);
|
||||
if (counter == 0) {
|
||||
builder->Abandon(); // Nothing to save
|
||||
} else {
|
||||
s = builder->Finish();
|
||||
if (s.ok()) {
|
||||
t.meta.file_size = builder->FileSize();
|
||||
}
|
||||
}
|
||||
delete builder;
|
||||
builder = NULL;
|
||||
|
||||
if (s.ok()) {
|
||||
s = file->Close();
|
||||
}
|
||||
delete file;
|
||||
file = NULL;
|
||||
|
||||
if (counter > 0 && s.ok()) {
|
||||
std::string orig = TableFileName(dbname_, t.meta.number);
|
||||
s = env_->RenameFile(copy, orig);
|
||||
if (s.ok()) {
|
||||
Log(options_.info_log, "Table #%llu: %d entries repaired",
|
||||
(unsigned long long) t.meta.number, counter);
|
||||
tables_.push_back(t);
|
||||
}
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env_->DeleteFile(copy);
|
||||
}
|
||||
}
|
||||
|
||||
Status WriteDescriptor() {
|
||||
std::string tmp = TempFileName(dbname_, 1);
|
||||
WritableFile* file;
|
||||
Status status = env_->NewWritableFile(tmp, &file, 4096);
|
||||
Status status = env_->NewWritableFile(tmp, &file);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
|
||||
SequenceNumber max_sequence = 0;
|
||||
for (int level=0; level<config::kNumLevels;++level)
|
||||
{
|
||||
std::vector<TableInfo> * table_ptr;
|
||||
std::vector<TableInfo>::const_iterator i;
|
||||
|
||||
table_ptr=&tables_[level];
|
||||
|
||||
for ( i = table_ptr->begin(); table_ptr->end()!= i; i++) {
|
||||
if (max_sequence < i->max_sequence) {
|
||||
max_sequence = i->max_sequence;
|
||||
}
|
||||
} // for
|
||||
} // for
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
if (max_sequence < tables_[i].max_sequence) {
|
||||
max_sequence = tables_[i].max_sequence;
|
||||
}
|
||||
}
|
||||
|
||||
edit_.SetComparatorName(icmp_.user_comparator()->Name());
|
||||
edit_.SetLogNumber(0);
|
||||
edit_.SetNextFile(next_file_number_);
|
||||
edit_.SetLastSequence(max_sequence);
|
||||
|
||||
for (int level=0; level<config::kNumLevels;++level)
|
||||
{
|
||||
std::vector<TableInfo> * table_ptr;
|
||||
std::vector<TableInfo>::const_iterator i;
|
||||
|
||||
table_ptr=&tables_[level];
|
||||
|
||||
for ( i = table_ptr->begin(); table_ptr->end()!= i; i++) {
|
||||
edit_.AddFile2(level, i->meta.number, i->meta.file_size,
|
||||
i->meta.smallest, i->meta.largest,
|
||||
i->meta.exp_write_low, i->meta.exp_write_high, i->meta.exp_explicit_high);
|
||||
|
||||
} // for
|
||||
} // for
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
// TODO(opt): separate out into multiple levels
|
||||
const TableInfo& t = tables_[i];
|
||||
edit_.AddFile(0, t.meta.number, t.meta.file_size,
|
||||
t.meta.smallest, t.meta.largest);
|
||||
}
|
||||
|
||||
//fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
|
||||
{
|
||||
log::Writer log(file);
|
||||
std::string record;
|
||||
edit_.EncodeTo(&record); // manifest format is default for release, options_ often incomplete
|
||||
edit_.EncodeTo(&record);
|
||||
status = log.AddRecord(record);
|
||||
}
|
||||
if (status.ok()) {
|
||||
@@ -483,33 +431,21 @@ class Repairer {
|
||||
return status;
|
||||
}
|
||||
|
||||
void ArchiveFile(const std::string& fname, bool two_levels=false) {
|
||||
void ArchiveFile(const std::string& fname) {
|
||||
// Move into another directory. E.g., for
|
||||
// dir/foo
|
||||
// rename to
|
||||
// dir/lost/foo
|
||||
std::string::size_type slash, slash2;
|
||||
|
||||
slash=fname.rfind('/');
|
||||
if (two_levels && std::string::npos!=slash && 0<slash)
|
||||
{
|
||||
slash2=fname.rfind('/',slash-1);
|
||||
if (std::string::npos==slash2)
|
||||
slash2=slash;
|
||||
} // if
|
||||
else
|
||||
slash2=slash;
|
||||
|
||||
const char* slash = strrchr(fname.c_str(), '/');
|
||||
std::string new_dir;
|
||||
|
||||
if (std::string::npos != slash2 && 0<slash2)
|
||||
new_dir.append(fname,0,slash2);
|
||||
|
||||
if (slash != NULL) {
|
||||
new_dir.assign(fname.data(), slash - fname.data());
|
||||
}
|
||||
new_dir.append("/lost");
|
||||
env_->CreateDir(new_dir); // Ignore error
|
||||
std::string new_file = new_dir;
|
||||
new_file.append("/");
|
||||
new_file.append((std::string::npos!=slash) ? fname.substr(slash+1) : fname);
|
||||
new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
|
||||
Status s = env_->RenameFile(fname, new_file);
|
||||
Log(options_.info_log, "Archiving %s: %s\n",
|
||||
fname.c_str(), s.ToString().c_str());
|
||||
|
||||
+13
-245
@@ -1,7 +1,10 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
#define STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
|
||||
// Thread safety
|
||||
// -------------
|
||||
//
|
||||
@@ -52,12 +55,6 @@ class SkipList {
|
||||
// Returns true iff an entry that compares equal to key is in the list.
|
||||
bool Contains(const Key& key) const;
|
||||
|
||||
// Returns true if all inserts have been sequentially increasing;
|
||||
// else this SkipList has had keys inserted in non-sequential order
|
||||
bool InSequentialInsertMode() const {
|
||||
return sequentialInsertMode_;
|
||||
}
|
||||
|
||||
// Iteration over the contents of a skip list
|
||||
class Iterator {
|
||||
public:
|
||||
@@ -97,22 +94,8 @@ class SkipList {
|
||||
// Intentionally copyable
|
||||
};
|
||||
|
||||
protected:
|
||||
// Checks the structure of this SkipList object, ensuring the keys are
|
||||
// properly ordered
|
||||
//
|
||||
// This is protected since it is intended for use by unit tests; if a lock
|
||||
// is used to protect Insert(), then it should be used to protect this
|
||||
// method as well
|
||||
bool Valid() const;
|
||||
|
||||
// Disables the sequential insert optimizations (used in performance testing)
|
||||
void DisableSequentialInsertMode() {
|
||||
sequentialInsertMode_ = false;
|
||||
}
|
||||
|
||||
private:
|
||||
enum { kMaxHeight = 17 };
|
||||
enum { kMaxHeight = 12 };
|
||||
|
||||
// Immutable after construction
|
||||
Comparator const compare_;
|
||||
@@ -132,18 +115,6 @@ class SkipList {
|
||||
// Read/written only by Insert().
|
||||
Random rnd_;
|
||||
|
||||
// Points to the last node in the list; modified only by Insert()
|
||||
Node* tail_;
|
||||
|
||||
// Pointers to the nodes previous to the tail node; have max_height_ entries
|
||||
Node* tailPrev_[kMaxHeight];
|
||||
|
||||
// The height of the tail_ node
|
||||
int tailHeight_;
|
||||
|
||||
// We track the tail node until we have a non-sequential insert
|
||||
bool sequentialInsertMode_;
|
||||
|
||||
Node* NewNode(const Key& key, int height);
|
||||
int RandomHeight();
|
||||
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
|
||||
@@ -158,11 +129,6 @@ class SkipList {
|
||||
// node at "level" for every level in [0..max_height_-1].
|
||||
Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
|
||||
|
||||
// Similar to FindGreaterOrEqual() except it uses the barrier-free
|
||||
// variant of Next(); this is used only by Insert() and it
|
||||
// checks the tail_ pointer in case we're doing a sequential insert
|
||||
Node* NoBarrier_FindGreaterOrEqual(const Key& key, Node** prev) const;
|
||||
|
||||
// Return the latest node with a key < key.
|
||||
// Return head_ if there is no such node.
|
||||
Node* FindLessThan(const Key& key) const;
|
||||
@@ -314,54 +280,6 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
typename SkipList<Key,Comparator>::Node*
|
||||
SkipList<Key,Comparator>::NoBarrier_FindGreaterOrEqual(const Key& key, Node** prev) const {
|
||||
int level = GetMaxHeight() - 1;
|
||||
|
||||
// If we have only seen sequential inserts up to this point, we can use
|
||||
// the tail_ node
|
||||
if ( sequentialInsertMode_ ) {
|
||||
if (tail_ == NULL) {
|
||||
// The list is currently empty, so the node being inserted
|
||||
// will be the new tail_
|
||||
assert(level == 0);
|
||||
if (prev != NULL) prev[0] = head_;
|
||||
return NULL;
|
||||
}
|
||||
else if (KeyIsAfterNode(key, tail_)) {
|
||||
// The new key must be inserted after the current tail_ node
|
||||
if (prev != NULL) {
|
||||
int i;
|
||||
for (i = 0; i < tailHeight_; ++i) {
|
||||
prev[i] = tail_;
|
||||
}
|
||||
for (/*continue with i*/; i <= level; ++i) {
|
||||
prev[i] = tailPrev_[i];
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
Node* x = head_;
|
||||
while (true) {
|
||||
Node* next = x->NoBarrier_Next(level);
|
||||
if (KeyIsAfterNode(key, next)) {
|
||||
// Keep searching in this list
|
||||
x = next;
|
||||
} else {
|
||||
if (prev != NULL) prev[level] = x;
|
||||
if (level == 0) {
|
||||
return next;
|
||||
} else {
|
||||
// Switch to next list
|
||||
level--;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
typename SkipList<Key,Comparator>::Node*
|
||||
SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
|
||||
@@ -409,41 +327,25 @@ SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
|
||||
arena_(arena),
|
||||
head_(NewNode(0 /* any key will do */, kMaxHeight)),
|
||||
max_height_(reinterpret_cast<void*>(1)),
|
||||
rnd_(0xdeadbeef),
|
||||
tail_(NULL),
|
||||
tailHeight_(0),
|
||||
sequentialInsertMode_(true) {
|
||||
rnd_(0xdeadbeef) {
|
||||
for (int i = 0; i < kMaxHeight; i++) {
|
||||
head_->SetNext(i, NULL);
|
||||
tailPrev_[i] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
void SkipList<Key,Comparator>::Insert(const Key& key) {
|
||||
// We use a barrier-free variant of FindGreaterOrEqual()
|
||||
// TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
|
||||
// here since Insert() is externally synchronized.
|
||||
Node* prev[kMaxHeight];
|
||||
Node* x = NoBarrier_FindGreaterOrEqual(key, prev);
|
||||
|
||||
// If we're still in sequential-insert mode, check if the new node is being
|
||||
// inserted at the end of the list, which is indicated by x being NULL
|
||||
if (sequentialInsertMode_) {
|
||||
if (x != NULL) {
|
||||
// we have a non-sequential (AKA random) insert, so stop maintaining
|
||||
// the tail bookkeeping overhead
|
||||
sequentialInsertMode_ = false;
|
||||
}
|
||||
}
|
||||
Node* x = FindGreaterOrEqual(key, prev);
|
||||
|
||||
// Our data structure does not allow duplicate insertion
|
||||
assert(x == NULL || !Equal(key, x->key));
|
||||
|
||||
int i, height = RandomHeight();
|
||||
int height = RandomHeight();
|
||||
if (height > GetMaxHeight()) {
|
||||
// We are extending max_height_ which means we need to fill in the blanks
|
||||
// in prev[] that were not filled in by NoBarrier_FindGreaterOrEqual()
|
||||
for (i = GetMaxHeight(); i < height; ++i) {
|
||||
for (int i = GetMaxHeight(); i < height; i++) {
|
||||
prev[i] = head_;
|
||||
}
|
||||
//fprintf(stderr, "Change height from %d to %d\n", max_height_, height);
|
||||
@@ -459,37 +361,12 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
|
||||
}
|
||||
|
||||
x = NewNode(key, height);
|
||||
for (i = 0; i < height; ++i) {
|
||||
for (int i = 0; i < height; i++) {
|
||||
// NoBarrier_SetNext() suffices since we will add a barrier when
|
||||
// we publish a pointer to "x" in prev[i].
|
||||
x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i));
|
||||
prev[i]->SetNext(i, x);
|
||||
}
|
||||
|
||||
// Do we need to update our tail_ pointer?
|
||||
if (sequentialInsertMode_) {
|
||||
Node* prevTail = tail_;
|
||||
int prevTailHeight = tailHeight_;
|
||||
|
||||
tail_ = x;
|
||||
tailHeight_ = height;
|
||||
|
||||
// We also need to update our tailPrev_ pointers; first we capture
|
||||
// the nodes already pointing to the new tail_
|
||||
for (i = 0; i < height; ++i) {
|
||||
tailPrev_[i] = prev[i];
|
||||
}
|
||||
|
||||
// If the previous tail node was taller than the new tail node, then
|
||||
// the prev pointers above the current tail node's height (up to the
|
||||
// height of the previous tail node) are simply the previous tail node
|
||||
for (/*continue with i*/; i < prevTailHeight; ++i) {
|
||||
tailPrev_[i] = prevTail;
|
||||
}
|
||||
|
||||
// NOTE: any prev pointers above prevTailHeight (up to max_height_) were
|
||||
// already set in tailPrev_ by previous calls to this method
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
@@ -502,115 +379,6 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
bool SkipList<Key,Comparator>::Valid() const
|
||||
{
|
||||
// Note that we can use barrier-free overloads in this method since it is
|
||||
// protected by the same lock as Insert().
|
||||
|
||||
// Ensure that the list is properly sorted; use an iterator for this check
|
||||
const Key* pPrevKey = NULL;
|
||||
typename SkipList<Key, Comparator>::Iterator iter(this);
|
||||
for ( iter.SeekToFirst(); iter.Valid(); iter.Next() ) {
|
||||
if ( pPrevKey != NULL ) {
|
||||
if ( compare_( *pPrevKey, iter.key() ) >= 0 ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
pPrevKey = &iter.key();
|
||||
}
|
||||
|
||||
// Now walk the linked list at each level and ensure it's sorted. Also track
|
||||
// how many nodes we see at each level; the number of nodes in the linked
|
||||
// list at level n must not be larger than the number of nodes at level n-1.
|
||||
std::vector<int> nodeCounts( GetMaxHeight() );
|
||||
int level;
|
||||
for ( level = GetMaxHeight() - 1; level >= 0; --level ) {
|
||||
int nodeCount = 0;
|
||||
pPrevKey = NULL;
|
||||
for ( Node* pNode = head_->NoBarrier_Next( level );
|
||||
pNode != NULL;
|
||||
pNode = pNode->NoBarrier_Next( level ) ) {
|
||||
++nodeCount;
|
||||
if ( pPrevKey != NULL ) {
|
||||
if ( compare_( *pPrevKey, pNode->key ) >= 0 ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
pPrevKey = &pNode->key;
|
||||
}
|
||||
nodeCounts[ level ] = nodeCount;
|
||||
}
|
||||
|
||||
// Ensure the node counts do not increase as we move up the levels
|
||||
int prevNodeCount = nodeCounts[0];
|
||||
for ( level = 1; level < GetMaxHeight(); ++level ) {
|
||||
int currentNodeCount = nodeCounts[ level ];
|
||||
if ( currentNodeCount > prevNodeCount ) {
|
||||
return false;
|
||||
}
|
||||
prevNodeCount = currentNodeCount;
|
||||
}
|
||||
|
||||
// Ensure that tail_ points to the last node
|
||||
if ( sequentialInsertMode_ ) {
|
||||
if ( tail_ == NULL ) {
|
||||
// tail_ is not set, so the list must be empty
|
||||
if ( tailPrev_[0] != NULL || head_->NoBarrier_Next(0) != NULL ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// we have a tail_ node; first ensure that its prev pointer actually
|
||||
// points to it
|
||||
if ( tailPrev_[0] == NULL || tailPrev_[0]->NoBarrier_Next(0) != tail_ ) {
|
||||
return false;
|
||||
}
|
||||
if ( compare_( tailPrev_[0]->key, tail_->key ) >= 0 ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// now check the rest of the pointers in tailPrev_; up to tailHeight_,
|
||||
// the next pointer of the node in tailPrev_ should point to tail_; after
|
||||
// that, the next pointer should be NULL
|
||||
for ( level = 1; level < GetMaxHeight(); ++level ) {
|
||||
Node* tailPrev = tailPrev_[ level ];
|
||||
if ( tailPrev == NULL ) {
|
||||
return false;
|
||||
}
|
||||
if ( level < tailHeight_ ) {
|
||||
if ( tailPrev->NoBarrier_Next( level ) != tail_ ) {
|
||||
return false;
|
||||
}
|
||||
if ( compare_( tailPrev->key, tail_->key ) >= 0 ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if ( tailPrev->NoBarrier_Next( level ) != NULL ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// the remainder of the tailPrev_ pointers (above max_height_)
|
||||
// should be NULL
|
||||
for ( /*continue with level*/; level < kMaxHeight; ++level ) {
|
||||
if ( tailPrev_[ level ] != NULL ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// now ensure that FindLast() returns tail_
|
||||
Node* lastNode = FindLast();
|
||||
if ( lastNode != tail_ ) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we get here, all is good
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
+11
-233
@@ -2,15 +2,11 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#include <inttypes.h>
|
||||
|
||||
#include "db/skiplist.h"
|
||||
#include <set>
|
||||
#include "leveldb/env.h"
|
||||
#include "util/arena.h"
|
||||
#include "util/hash.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
@@ -30,29 +26,15 @@ struct Comparator {
|
||||
}
|
||||
};
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
class SkipListTest : public SkipList<Key, Comparator>
|
||||
{
|
||||
public:
|
||||
SkipListTest(Comparator cmp, Arena* arena) : SkipList<Key, Comparator>(cmp, arena) {}
|
||||
|
||||
// check the validity of this SkipList object by calling the Valid() method
|
||||
// in the base class
|
||||
bool Valid() const { return SkipList<Key, Comparator>::Valid(); }
|
||||
|
||||
void DisableSequentialInsertMode() { SkipList<Key, Comparator>::DisableSequentialInsertMode(); }
|
||||
};
|
||||
|
||||
class SkipTest { };
|
||||
|
||||
TEST(SkipTest, Empty) {
|
||||
Arena arena;
|
||||
Comparator cmp;
|
||||
SkipListTest<Key, Comparator> list(cmp, &arena);
|
||||
SkipList<Key, Comparator> list(cmp, &arena);
|
||||
ASSERT_TRUE(!list.Contains(10));
|
||||
ASSERT_TRUE(list.Valid());
|
||||
|
||||
SkipListTest<Key, Comparator>::Iterator iter(&list);
|
||||
SkipList<Key, Comparator>::Iterator iter(&list);
|
||||
ASSERT_TRUE(!iter.Valid());
|
||||
iter.SeekToFirst();
|
||||
ASSERT_TRUE(!iter.Valid());
|
||||
@@ -69,14 +51,13 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
std::set<Key> keys;
|
||||
Arena arena;
|
||||
Comparator cmp;
|
||||
SkipListTest<Key, Comparator> list(cmp, &arena);
|
||||
SkipList<Key, Comparator> list(cmp, &arena);
|
||||
for (int i = 0; i < N; i++) {
|
||||
Key key = rnd.Next() % R;
|
||||
if (keys.insert(key).second) {
|
||||
list.Insert(key);
|
||||
}
|
||||
}
|
||||
ASSERT_TRUE(list.Valid());
|
||||
|
||||
for (int i = 0; i < R; i++) {
|
||||
if (list.Contains(i)) {
|
||||
@@ -88,7 +69,7 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
|
||||
// Simple iterator tests
|
||||
{
|
||||
SkipListTest<Key, Comparator>::Iterator iter(&list);
|
||||
SkipList<Key, Comparator>::Iterator iter(&list);
|
||||
ASSERT_TRUE(!iter.Valid());
|
||||
|
||||
iter.Seek(0);
|
||||
@@ -106,7 +87,7 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
|
||||
// Forward iteration test
|
||||
for (int i = 0; i < R; i++) {
|
||||
SkipListTest<Key, Comparator>::Iterator iter(&list);
|
||||
SkipList<Key, Comparator>::Iterator iter(&list);
|
||||
iter.Seek(i);
|
||||
|
||||
// Compare against model iterator
|
||||
@@ -126,7 +107,7 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
|
||||
// Backward iteration test
|
||||
{
|
||||
SkipListTest<Key, Comparator>::Iterator iter(&list);
|
||||
SkipList<Key, Comparator>::Iterator iter(&list);
|
||||
iter.SeekToLast();
|
||||
|
||||
// Compare against model iterator
|
||||
@@ -269,7 +250,7 @@ class ConcurrentTest {
|
||||
// Note that generation 0 is never inserted, so it is ok if
|
||||
// <*,0,*> is missing.
|
||||
ASSERT_TRUE((gen(pos) == 0) ||
|
||||
(gen(pos) > initial_state.Get(key(pos)))
|
||||
(gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))
|
||||
) << "key: " << key(pos)
|
||||
<< "; gen: " << gen(pos)
|
||||
<< "; initgen: "
|
||||
@@ -332,16 +313,18 @@ class TestState {
|
||||
state_cv_(&mu_) {}
|
||||
|
||||
void Wait(ReaderState s) {
|
||||
MutexLock lock(&mu_);
|
||||
mu_.Lock();
|
||||
while (state_ != s) {
|
||||
state_cv_.Wait();
|
||||
}
|
||||
mu_.Unlock();
|
||||
}
|
||||
|
||||
void Change(ReaderState s) {
|
||||
MutexLock lock(&mu_);
|
||||
mu_.Lock();
|
||||
state_ = s;
|
||||
state_cv_.Signal();
|
||||
mu_.Unlock();
|
||||
}
|
||||
|
||||
private:
|
||||
@@ -388,211 +371,6 @@ TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
|
||||
TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
|
||||
TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
|
||||
|
||||
static void
|
||||
RunSequentialInsert(
|
||||
const int NumKeys,
|
||||
bool AcquireLock,
|
||||
bool ReverseInsert,
|
||||
bool SequentialInsertModeEnabled )
|
||||
{
|
||||
const int loopCount = 5; // repeat the whole process this many times and average the time spent
|
||||
std::vector<uint64_t> timeSpent;
|
||||
|
||||
port::Mutex mutex;
|
||||
Env* env = Env::Default();
|
||||
|
||||
fprintf( stderr,
|
||||
"Sequentially inserting %d keys in %s order,\n"
|
||||
" seqential insert mode is initially %sabled,\n"
|
||||
" %sacquiring a lock for each insert (averaging over %d runs)\n",
|
||||
NumKeys, ReverseInsert ? "reverse" : "forward",
|
||||
SequentialInsertModeEnabled ? "en" : "dis",
|
||||
AcquireLock ? "" : "not ", loopCount );
|
||||
|
||||
int k;
|
||||
for ( k = 0; k < loopCount; ++k ) {
|
||||
int j;
|
||||
Arena arena;
|
||||
Comparator cmp;
|
||||
SkipListTest<Key, Comparator> list( cmp, &arena );
|
||||
|
||||
// initially the SkipList should be in sequential mode
|
||||
ASSERT_TRUE( list.InSequentialInsertMode() );
|
||||
|
||||
// were we instructed to disable sequential insert mode?
|
||||
if ( !SequentialInsertModeEnabled ) {
|
||||
list.DisableSequentialInsertMode();
|
||||
ASSERT_TRUE( !list.InSequentialInsertMode() );
|
||||
}
|
||||
|
||||
uint64_t start = env->NowMicros();
|
||||
for ( j = 0; j < NumKeys; ++j ) {
|
||||
Key key = ReverseInsert ? NumKeys - 1 - j : j;
|
||||
|
||||
if ( AcquireLock ) mutex.Lock();
|
||||
list.Insert( key );
|
||||
if ( AcquireLock ) mutex.Unlock();
|
||||
}
|
||||
uint64_t stop = env->NowMicros();
|
||||
timeSpent.push_back( stop - start );
|
||||
//fprintf( stderr, " Time for run %d: %llu\n", k, timeSpent[k] );
|
||||
|
||||
// if SequentialInsertModeEnabled is true, the SkipList should still be
|
||||
// in sequential mode iff ReverseInsert is false
|
||||
if ( SequentialInsertModeEnabled ) {
|
||||
ASSERT_TRUE( list.InSequentialInsertMode() != ReverseInsert );
|
||||
}
|
||||
else {
|
||||
ASSERT_TRUE( !list.InSequentialInsertMode() );
|
||||
}
|
||||
|
||||
// ensure the SkipLlist is properly sorted
|
||||
if ( AcquireLock ) mutex.Lock();
|
||||
ASSERT_TRUE( list.Valid() );
|
||||
if ( AcquireLock ) mutex.Unlock();
|
||||
|
||||
// ensure the SkipList contains all the keys we inserted
|
||||
for ( j = 0; j < NumKeys; ++j ) {
|
||||
ASSERT_TRUE( list.Contains( j ) );
|
||||
}
|
||||
}
|
||||
|
||||
// throw out the low and high times and average the rest
|
||||
uint64_t totalTime, lowTime, highTime;
|
||||
totalTime = lowTime = highTime = timeSpent[0];
|
||||
for ( k = 1; k < loopCount; ++k ) {
|
||||
uint64_t currentTime = timeSpent[k];
|
||||
totalTime += currentTime;
|
||||
if ( lowTime > currentTime ) lowTime = currentTime;
|
||||
if ( highTime < currentTime ) highTime = currentTime;
|
||||
}
|
||||
|
||||
totalTime -= (lowTime + highTime);
|
||||
|
||||
uint64_t averageTime = (totalTime / (loopCount - 2));
|
||||
double timePerKey = (double)averageTime / (double)NumKeys;
|
||||
fprintf( stderr, " Average insertion time: %" PRIu64 " (%f/key)\n", averageTime, timePerKey );
|
||||
}
|
||||
|
||||
TEST(SkipTest, SequentialInsert_NoLock_ForwardInsert)
|
||||
{
|
||||
int numKeys = 100000;
|
||||
bool acquireLock = false;
|
||||
bool reverseInsert = false;
|
||||
bool sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
sequentialInsertModeEnabled = false;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
}
|
||||
|
||||
TEST(SkipTest, SequentialInsert_Lock_ForwardInsert)
|
||||
{
|
||||
int numKeys = 100000;
|
||||
bool acquireLock = true;
|
||||
bool reverseInsert = false;
|
||||
bool sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
sequentialInsertModeEnabled = false;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
}
|
||||
|
||||
TEST(SkipTest, SequentialInsert_NoLock_ReverseInsert)
|
||||
{
|
||||
int numKeys = 100000;
|
||||
bool acquireLock = false;
|
||||
bool reverseInsert = true;
|
||||
bool sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
}
|
||||
|
||||
TEST(SkipTest, SequentialInsert_Lock_ReverseInsert)
|
||||
{
|
||||
int numKeys = 100000;
|
||||
bool acquireLock = true;
|
||||
bool reverseInsert = true;
|
||||
bool sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
}
|
||||
|
||||
TEST(SkipTest, SequentialInsert_IncreasingNumberOfInserts)
|
||||
{
|
||||
// test with increasing numbers of keys, with sequential-insert mode both
|
||||
// enabled and disabled; we're looking to see if per-key insertion times
|
||||
// trend upward as the number of keys increases
|
||||
int numKeys = 10000;
|
||||
bool acquireLock = false;
|
||||
bool reverseInsert = false;
|
||||
bool sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
sequentialInsertModeEnabled = false;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
numKeys = 100000;
|
||||
sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
sequentialInsertModeEnabled = false;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
numKeys = 1000000;
|
||||
sequentialInsertModeEnabled = true;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
|
||||
sequentialInsertModeEnabled = false;
|
||||
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
|
||||
}
|
||||
|
||||
TEST(SkipTest, SequentialInsert_MixedInsertionModes)
|
||||
{
|
||||
// start inserting sequentially, then switch to non-sequential inserts,
|
||||
// ensuring all works as intended
|
||||
int j, numSequentialKeys = 100000, numNonSequentialKeys = 100000;
|
||||
int totalNumKeys = numSequentialKeys + numNonSequentialKeys;
|
||||
Arena arena;
|
||||
Comparator cmp;
|
||||
SkipListTest<Key, Comparator> list( cmp, &arena );
|
||||
|
||||
// initially the SkipList should be in sequential mode
|
||||
ASSERT_TRUE( list.InSequentialInsertMode() );
|
||||
|
||||
// start inserting at key=1; when we insert 0 below, the list should switch
|
||||
// out of sequential insert mode
|
||||
for ( j = 1; j < numSequentialKeys; ++j ) {
|
||||
list.Insert( j );
|
||||
}
|
||||
|
||||
// the SkipList should still be in sequential mode
|
||||
ASSERT_TRUE( list.InSequentialInsertMode() );
|
||||
ASSERT_TRUE( list.Valid() );
|
||||
|
||||
list.Insert( 0 );
|
||||
ASSERT_TRUE( !list.InSequentialInsertMode() );
|
||||
ASSERT_TRUE( list.Valid() );
|
||||
|
||||
// now insert the remaining keys in non-sequential order (they're not
|
||||
// random, but that doesn't matter here; just ensure we switch to
|
||||
// non-sequential mode and that all continues to work)
|
||||
for ( j = 0; j < numNonSequentialKeys; j += 2 ) {
|
||||
int key = totalNumKeys - j - 1;
|
||||
list.Insert( key );
|
||||
}
|
||||
for ( j = 0; j < numNonSequentialKeys; j += 2 ) {
|
||||
int key = numSequentialKeys + j;
|
||||
list.Insert( key );
|
||||
}
|
||||
|
||||
ASSERT_TRUE( !list.InSequentialInsertMode() );
|
||||
ASSERT_TRUE( list.Valid() );
|
||||
|
||||
// ensure the SkipList contains all the keys we inserted
|
||||
for ( j = 0; j < totalNumKeys; ++j ) {
|
||||
ASSERT_TRUE( list.Contains( j ) );
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/db.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
+25
-134
@@ -5,26 +5,22 @@
|
||||
#include "db/table_cache.h"
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "util/coding.h"
|
||||
#include "leveldb/perf_count.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
struct TableAndFile {
|
||||
RandomAccessFile* file;
|
||||
Table* table;
|
||||
};
|
||||
|
||||
static void DeleteEntry(const Slice& key, void* value) {
|
||||
TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
|
||||
if (0==dec_and_fetch(&tf->user_count))
|
||||
{
|
||||
if (NULL!=tf->doublecache)
|
||||
tf->doublecache->SubFileSize(tf->table->GetFileSize());
|
||||
delete tf->table;
|
||||
delete tf->file;
|
||||
delete tf;
|
||||
} // if
|
||||
delete tf->table;
|
||||
delete tf->file;
|
||||
delete tf;
|
||||
}
|
||||
|
||||
static void UnrefEntry(void* arg1, void* arg2) {
|
||||
@@ -35,38 +31,37 @@ static void UnrefEntry(void* arg1, void* arg2) {
|
||||
|
||||
TableCache::TableCache(const std::string& dbname,
|
||||
const Options* options,
|
||||
Cache * file_cache,
|
||||
DoubleCache & doublecache)
|
||||
int entries)
|
||||
: env_(options->env),
|
||||
dbname_(dbname),
|
||||
options_(options),
|
||||
cache_(file_cache),
|
||||
doublecache_(doublecache)
|
||||
{
|
||||
cache_(NewLRUCache(entries)) {
|
||||
}
|
||||
|
||||
TableCache::~TableCache() {
|
||||
delete cache_;
|
||||
}
|
||||
|
||||
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, int level,
|
||||
Cache::Handle** handle, bool is_compaction,
|
||||
bool for_iterator) {
|
||||
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
|
||||
Cache::Handle** handle) {
|
||||
Status s;
|
||||
char buf[sizeof(file_number)];
|
||||
EncodeFixed64(buf, file_number);
|
||||
Slice key(buf, sizeof(buf));
|
||||
*handle = cache_->Lookup(key);
|
||||
if (*handle == NULL) {
|
||||
std::string fname = TableFileName(*options_, file_number, level);
|
||||
std::string fname = TableFileName(dbname_, file_number);
|
||||
RandomAccessFile* file = NULL;
|
||||
Table* table = NULL;
|
||||
s = env_->NewRandomAccessFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
std::string old_fname = SSTTableFileName(dbname_, file_number);
|
||||
if (env_->NewRandomAccessFile(old_fname, &file).ok()) {
|
||||
s = Status::OK();
|
||||
}
|
||||
}
|
||||
if (s.ok()) {
|
||||
s = Table::Open(*options_, file, file_size, &table);
|
||||
|
||||
// Riak: support opportunity to manage Linux page cache
|
||||
if (is_compaction)
|
||||
file->SetForCompaction(file_size);
|
||||
}
|
||||
|
||||
if (!s.ok()) {
|
||||
@@ -78,74 +73,22 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, int level
|
||||
TableAndFile* tf = new TableAndFile;
|
||||
tf->file = file;
|
||||
tf->table = table;
|
||||
tf->doublecache = &doublecache_;
|
||||
tf->file_number = file_number;
|
||||
tf->level = level;
|
||||
|
||||
*handle = cache_->Insert(key, tf, table->TableObjectSize(), &DeleteEntry);
|
||||
gPerfCounters->Inc(ePerfTableOpened);
|
||||
doublecache_.AddFileSize(table->GetFileSize());
|
||||
|
||||
// temporary hardcoding to match number of levels defined as
|
||||
// overlapped in version_set.cc
|
||||
if (level<config::kNumOverlapLevels)
|
||||
cache_->Addref(*handle);
|
||||
*handle = cache_->Insert(key, tf, 1, &DeleteEntry);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Table *table = reinterpret_cast<TableAndFile*>(cache_->Value(*handle))->table;
|
||||
|
||||
// this is NOT first access, see if bloom filter can load now
|
||||
if (!for_iterator && table->ReadFilter())
|
||||
{
|
||||
// TableAndFile now going to be present in two cache entries
|
||||
// 1. retrieve old entry within file cache
|
||||
TableAndFile* tf = reinterpret_cast<TableAndFile*>(cache_->Value(*handle));
|
||||
inc_and_fetch(&tf->user_count);
|
||||
|
||||
// 2. must clean file size, do not want double count
|
||||
if (NULL!=tf->doublecache)
|
||||
tf->doublecache->SubFileSize(tf->table->GetFileSize());
|
||||
|
||||
// 3. release current reference (and possible special overlap reference)
|
||||
cache_->Release(*handle);
|
||||
if (tf->level<config::kNumOverlapLevels)
|
||||
cache_->Release(*handle);
|
||||
|
||||
// 4. create second table cache entry using TableObjectSize that now includes
|
||||
// bloom filter size
|
||||
*handle = cache_->Insert(key, tf, table->TableObjectSize(), &DeleteEntry);
|
||||
|
||||
// 5. set double reference if an overlapped file (prevents from being flushed)
|
||||
if (level<config::kNumOverlapLevels)
|
||||
cache_->Addref(*handle);
|
||||
} // if
|
||||
|
||||
// for Linux, let fadvise start precaching
|
||||
if (is_compaction)
|
||||
{
|
||||
RandomAccessFile *file = reinterpret_cast<TableAndFile*>(cache_->Value(*handle))->file;
|
||||
file->SetForCompaction(file_size);
|
||||
} // if
|
||||
|
||||
gPerfCounters->Inc(ePerfTableCached);
|
||||
} // else
|
||||
return s;
|
||||
}
|
||||
|
||||
Iterator* TableCache::NewIterator(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
int level,
|
||||
Table** tableptr) {
|
||||
if (tableptr != NULL) {
|
||||
*tableptr = NULL;
|
||||
}
|
||||
|
||||
Cache::Handle* handle = NULL;
|
||||
Status s = FindTable(file_number, file_size, level, &handle, options.IsCompaction(), true);
|
||||
|
||||
Status s = FindTable(file_number, file_size, &handle);
|
||||
if (!s.ok()) {
|
||||
return NewErrorIterator(s);
|
||||
}
|
||||
@@ -162,13 +105,11 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
|
||||
Status TableCache::Get(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
int level,
|
||||
const Slice& k,
|
||||
void* arg,
|
||||
bool (*saver)(void*, const Slice&, const Slice&)) {
|
||||
void (*saver)(void*, const Slice&, const Slice&)) {
|
||||
Cache::Handle* handle = NULL;
|
||||
Status s = FindTable(file_number, file_size, level, &handle);
|
||||
|
||||
Status s = FindTable(file_number, file_size, &handle);
|
||||
if (s.ok()) {
|
||||
Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
|
||||
s = t->InternalGet(options, k, arg, saver);
|
||||
@@ -177,60 +118,10 @@ Status TableCache::Get(const ReadOptions& options,
|
||||
return s;
|
||||
}
|
||||
|
||||
void TableCache::Evict(uint64_t file_number, bool is_overlapped) {
|
||||
void TableCache::Evict(uint64_t file_number) {
|
||||
char buf[sizeof(file_number)];
|
||||
EncodeFixed64(buf, file_number);
|
||||
|
||||
// overlapped files have extra reference to prevent their purge,
|
||||
// release that reference now
|
||||
if (is_overlapped)
|
||||
{
|
||||
Cache::Handle *handle;
|
||||
|
||||
// the Lookup call adds a reference too, back out both
|
||||
handle=cache_->Lookup(Slice(buf, sizeof(buf)));
|
||||
|
||||
// with multiple background threads, file might already be
|
||||
// evicted
|
||||
if (NULL!=handle)
|
||||
{
|
||||
cache_->Release(handle); // release for Lookup() call just made
|
||||
cache_->Release(handle); // release for extra reference
|
||||
} // if
|
||||
} // if
|
||||
|
||||
cache_->Erase(Slice(buf, sizeof(buf)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Riak specific routine to return table statistic ONLY if table metadata
|
||||
* already within cache ... otherwise return 0.
|
||||
*/
|
||||
uint64_t
|
||||
TableCache::GetStatisticValue(
|
||||
uint64_t file_number,
|
||||
unsigned Index)
|
||||
{
|
||||
uint64_t ret_val;
|
||||
char buf[sizeof(file_number)];
|
||||
Cache::Handle *handle;
|
||||
|
||||
ret_val=0;
|
||||
EncodeFixed64(buf, file_number);
|
||||
Slice key(buf, sizeof(buf));
|
||||
handle = cache_->Lookup(key);
|
||||
|
||||
if (NULL != handle)
|
||||
{
|
||||
TableAndFile * tf;
|
||||
|
||||
tf=reinterpret_cast<TableAndFile*>(cache_->Value(handle));
|
||||
ret_val=tf->table->GetSstCounters().Value(Index);
|
||||
cache_->Release(handle);
|
||||
} // if
|
||||
|
||||
return(ret_val);
|
||||
|
||||
} // TableCache::GetStatisticValue
|
||||
|
||||
} // namespace leveldb
|
||||
@@ -13,7 +13,6 @@
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "port/port.h"
|
||||
#include "util/cache2.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@@ -21,10 +20,8 @@ class Env;
|
||||
|
||||
class TableCache {
|
||||
public:
|
||||
// clean up note: file_cache is redundant to GetFileCache available from doublecache
|
||||
TableCache(const std::string& dbname, const Options* options, Cache * file_cache,
|
||||
DoubleCache & doublecache);
|
||||
virtual ~TableCache();
|
||||
TableCache(const std::string& dbname, const Options* options, int entries);
|
||||
~TableCache();
|
||||
|
||||
// Return an iterator for the specified file number (the corresponding
|
||||
// file length must be exactly "file_size" bytes). If "tableptr" is
|
||||
@@ -36,7 +33,6 @@ class TableCache {
|
||||
Iterator* NewIterator(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
int level,
|
||||
Table** tableptr = NULL);
|
||||
|
||||
// If a seek to internal key "k" in specified file finds an entry,
|
||||
@@ -44,65 +40,22 @@ class TableCache {
|
||||
Status Get(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
int level,
|
||||
const Slice& k,
|
||||
void* arg,
|
||||
bool (*handle_result)(void*, const Slice&, const Slice&));
|
||||
void (*handle_result)(void*, const Slice&, const Slice&));
|
||||
|
||||
// Evict any entry for the specified file number
|
||||
void Evict(uint64_t file_number, bool is_overlapped);
|
||||
void Evict(uint64_t file_number);
|
||||
|
||||
// Riak specific: return table statistic ONLY if table in cache, otherwise zero
|
||||
uint64_t GetStatisticValue(uint64_t file_number, unsigned Index);
|
||||
|
||||
|
||||
// access for testing tools, not for public access
|
||||
Status TEST_FindTable(uint64_t file_number, uint64_t file_size, int level, Cache::Handle** handle)
|
||||
{return( FindTable(file_number, file_size, level, handle));};
|
||||
|
||||
Cache* TEST_GetInternalCache() {return(cache_);};
|
||||
|
||||
void Release(Cache::Handle * handle) {cache_->Release(handle);};
|
||||
|
||||
// routine called if Options::cache_object_warming is true.
|
||||
// Writes list of all file names currently in file cache to disk.
|
||||
Status SaveOpenFileList();
|
||||
|
||||
// routine called if Options::cache_object_warming is true.
|
||||
// Reads file created by SaveOpenFileList() and attempts to open
|
||||
// every file.
|
||||
Status PreloadTableCache();
|
||||
|
||||
// was private, now protected to allow easy unit test overrides
|
||||
protected:
|
||||
private:
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
const Options* options_;
|
||||
Cache * cache_;
|
||||
DoubleCache & doublecache_;
|
||||
Cache* cache_;
|
||||
|
||||
// virtual to enable unit test overrides
|
||||
virtual Status FindTable(uint64_t file_number, uint64_t file_size, int level,
|
||||
Cache::Handle**, bool is_compaction=false,
|
||||
bool for_iterator=false);
|
||||
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
|
||||
};
|
||||
|
||||
|
||||
struct TableAndFile {
|
||||
RandomAccessFile* file;
|
||||
Table* table;
|
||||
DoubleCache * doublecache;
|
||||
uint64_t file_number; // saved for cache object warming
|
||||
int level; // saved for cache object warming
|
||||
volatile uint32_t user_count;
|
||||
|
||||
TableAndFile()
|
||||
: file(NULL), table(NULL), doublecache(NULL),
|
||||
file_number(0), level(0), user_count(1)
|
||||
{};
|
||||
};
|
||||
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
@@ -9,6 +9,20 @@
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
// Tag numbers for serialized VersionEdit. These numbers are written to
|
||||
// disk and should not be changed.
|
||||
enum Tag {
|
||||
kComparator = 1,
|
||||
kLogNumber = 2,
|
||||
kNextFileNumber = 3,
|
||||
kLastSequence = 4,
|
||||
kCompactPointer = 5,
|
||||
kDeletedFile = 6,
|
||||
kNewFile = 7,
|
||||
// 8 was used for large value refs
|
||||
kPrevLogNumber = 9
|
||||
};
|
||||
|
||||
void VersionEdit::Clear() {
|
||||
comparator_.clear();
|
||||
log_number_ = 0;
|
||||
@@ -20,21 +34,11 @@ void VersionEdit::Clear() {
|
||||
has_prev_log_number_ = false;
|
||||
has_next_file_number_ = false;
|
||||
has_last_sequence_ = false;
|
||||
has_f1_files_ = false;
|
||||
has_f2_files_ = false;
|
||||
|
||||
deleted_files_.clear();
|
||||
new_files_.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* EncodeTo serializes the VersionEdit object
|
||||
* to the "dst" string parameter. "format2" flag
|
||||
* indicates whether serialization should use original
|
||||
* Google format for file objects (false) or Basho's updated
|
||||
* file2 format for expiry enabled file objects (true)
|
||||
*/
|
||||
void VersionEdit::EncodeTo(std::string* dst, bool format2) const {
|
||||
void VersionEdit::EncodeTo(std::string* dst) const {
|
||||
if (has_comparator_) {
|
||||
PutVarint32(dst, kComparator);
|
||||
PutLengthPrefixedSlice(dst, comparator_);
|
||||
@@ -72,21 +76,12 @@ void VersionEdit::EncodeTo(std::string* dst, bool format2) const {
|
||||
|
||||
for (size_t i = 0; i < new_files_.size(); i++) {
|
||||
const FileMetaData& f = new_files_[i].second;
|
||||
if (format2)
|
||||
PutVarint32(dst, kNewFile2);
|
||||
else
|
||||
PutVarint32(dst, kNewFile);
|
||||
PutVarint32(dst, kNewFile);
|
||||
PutVarint32(dst, new_files_[i].first); // level
|
||||
PutVarint64(dst, f.number);
|
||||
PutVarint64(dst, f.file_size);
|
||||
PutLengthPrefixedSlice(dst, f.smallest.Encode());
|
||||
PutLengthPrefixedSlice(dst, f.largest.Encode());
|
||||
if (format2)
|
||||
{
|
||||
PutVarint64(dst, f.exp_write_low);
|
||||
PutVarint64(dst, f.exp_write_high);
|
||||
PutVarint64(dst, f.exp_explicit_high);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +98,7 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
|
||||
static bool GetLevel(Slice* input, int* level) {
|
||||
uint32_t v;
|
||||
if (GetVarint32(input, &v) &&
|
||||
v < (unsigned)config::kNumLevels) {
|
||||
v < config::kNumLevels) {
|
||||
*level = v;
|
||||
return true;
|
||||
} else {
|
||||
@@ -190,34 +185,13 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
GetVarint64(&input, &f.number) &&
|
||||
GetVarint64(&input, &f.file_size) &&
|
||||
GetInternalKey(&input, &f.smallest) &&
|
||||
GetInternalKey(&input, &f.largest))
|
||||
{
|
||||
has_f1_files_ = true;
|
||||
f.level=level;
|
||||
GetInternalKey(&input, &f.largest)) {
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
} else {
|
||||
msg = "new-file entry";
|
||||
}
|
||||
break;
|
||||
|
||||
case kNewFile2:
|
||||
if (GetLevel(&input, &level) &&
|
||||
GetVarint64(&input, &f.number) &&
|
||||
GetVarint64(&input, &f.file_size) &&
|
||||
GetInternalKey(&input, &f.smallest) &&
|
||||
GetInternalKey(&input, &f.largest) &&
|
||||
GetVarint64(&input, &f.exp_write_low) &&
|
||||
GetVarint64(&input, &f.exp_write_high) &&
|
||||
GetVarint64(&input, &f.exp_explicit_high))
|
||||
{
|
||||
has_f2_files_ = true;
|
||||
f.level=level;
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
} else {
|
||||
msg = "new-file2 entry";
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
msg = "unknown tag";
|
||||
break;
|
||||
@@ -284,12 +258,6 @@ std::string VersionEdit::DebugString() const {
|
||||
r.append(f.smallest.DebugString());
|
||||
r.append(" .. ");
|
||||
r.append(f.largest.DebugString());
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, f.exp_write_low);
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, f.exp_write_high);
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, f.exp_explicit_high);
|
||||
}
|
||||
r.append("\n}\n");
|
||||
return r;
|
||||
|
||||
@@ -16,41 +16,15 @@ class VersionSet;
|
||||
|
||||
struct FileMetaData {
|
||||
int refs;
|
||||
// int allowed_seeks; // Seeks allowed until compaction
|
||||
int allowed_seeks; // Seeks allowed until compaction
|
||||
uint64_t number;
|
||||
uint64_t file_size; // File size in bytes
|
||||
uint64_t num_entries; // count of values in .sst file, only valid during table build
|
||||
InternalKey smallest; // Smallest internal key served by table
|
||||
InternalKey largest; // Largest internal key served by table
|
||||
int level;
|
||||
ExpiryTimeMicros exp_write_low; // oldest write time in file:
|
||||
// 0 - non-expiry keys exist too
|
||||
// ULLONG_MAX - no write time expiry & no plain keys
|
||||
ExpiryTimeMicros exp_write_high; // most recent write time in file
|
||||
ExpiryTimeMicros exp_explicit_high; // most recent/furthest into future explicit expiry
|
||||
|
||||
FileMetaData()
|
||||
: refs(0), /*allowed_seeks(1 << 30),*/ file_size(0),
|
||||
num_entries(0), level(-1), exp_write_low(0), exp_write_high(0), exp_explicit_high(0)
|
||||
{ }
|
||||
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
|
||||
};
|
||||
|
||||
|
||||
class FileMetaDataPtrCompare
|
||||
{
|
||||
protected:
|
||||
const Comparator * comparator_;
|
||||
|
||||
public:
|
||||
explicit FileMetaDataPtrCompare(const Comparator * Comparer)
|
||||
: comparator_(Comparer) {};
|
||||
|
||||
bool operator() (const FileMetaData * file1, const FileMetaData * file2) const
|
||||
{
|
||||
return(comparator_->Compare(file1->smallest.user_key(), file2->smallest.user_key()) < 0);
|
||||
}
|
||||
}; // class FileMetaDataPtrCompare
|
||||
|
||||
class VersionEdit {
|
||||
public:
|
||||
VersionEdit() { Clear(); }
|
||||
@@ -85,7 +59,6 @@ class VersionEdit {
|
||||
// Add the specified file at the specified number.
|
||||
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
|
||||
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file
|
||||
#if 0
|
||||
void AddFile(int level, uint64_t file,
|
||||
uint64_t file_size,
|
||||
const InternalKey& smallest,
|
||||
@@ -95,27 +68,6 @@ class VersionEdit {
|
||||
f.file_size = file_size;
|
||||
f.smallest = smallest;
|
||||
f.largest = largest;
|
||||
f.level = level;
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
}
|
||||
#endif
|
||||
|
||||
void AddFile2(int level, uint64_t file,
|
||||
uint64_t file_size,
|
||||
const InternalKey& smallest,
|
||||
const InternalKey& largest,
|
||||
uint64_t exp_write_low,
|
||||
uint64_t exp_write_high,
|
||||
uint64_t exp_explicit_high) {
|
||||
FileMetaData f;
|
||||
f.number = file;
|
||||
f.file_size = file_size;
|
||||
f.smallest = smallest;
|
||||
f.largest = largest;
|
||||
f.level = level;
|
||||
f.exp_write_low = exp_write_low;
|
||||
f.exp_write_high = exp_write_high;
|
||||
f.exp_explicit_high = exp_explicit_high;
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
}
|
||||
|
||||
@@ -123,37 +75,16 @@ class VersionEdit {
|
||||
void DeleteFile(int level, uint64_t file) {
|
||||
deleted_files_.insert(std::make_pair(level, file));
|
||||
}
|
||||
size_t DeletedFileCount() const {return(deleted_files_.size());};
|
||||
|
||||
void EncodeTo(std::string* dst, bool format2=true) const;
|
||||
void EncodeTo(std::string* dst) const;
|
||||
Status DecodeFrom(const Slice& src);
|
||||
|
||||
// unit test access to validate file entries' format types
|
||||
bool HasF1Files() const {return(has_f1_files_);};
|
||||
bool HasF2Files() const {return(has_f2_files_);};
|
||||
|
||||
std::string DebugString() const;
|
||||
|
||||
// Tag numbers for serialized VersionEdit. These numbers are written to
|
||||
// disk and should not be changed.
|
||||
enum Tag {
|
||||
kComparator = 1,
|
||||
kLogNumber = 2,
|
||||
kNextFileNumber = 3,
|
||||
kLastSequence = 4,
|
||||
kCompactPointer = 5,
|
||||
kDeletedFile = 6,
|
||||
kNewFile = 7,
|
||||
// 8 was used for large value refs
|
||||
kPrevLogNumber = 9,
|
||||
kFileCacheObject = 10,
|
||||
kNewFile2 = 11 // expiry capable file
|
||||
};
|
||||
|
||||
private:
|
||||
friend class VersionSet;
|
||||
|
||||
USED_BY_NESTED_FRIEND2(typedef std::set< std::pair<int, uint64_t> > DeletedFileSet)
|
||||
typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
|
||||
|
||||
std::string comparator_;
|
||||
uint64_t log_number_;
|
||||
@@ -165,13 +96,10 @@ enum Tag {
|
||||
bool has_prev_log_number_;
|
||||
bool has_next_file_number_;
|
||||
bool has_last_sequence_;
|
||||
// following should be mutually exclusive, but tested independently to be sure
|
||||
bool has_f1_files_; // manifest uses format 1 (for unit tests)
|
||||
bool has_f2_files_; // manifest uses format 2 (for unit tests)
|
||||
|
||||
USED_BY_NESTED_FRIEND2(std::vector< std::pair<int, InternalKey> > compact_pointers_)
|
||||
USED_BY_NESTED_FRIEND(DeletedFileSet deleted_files_)
|
||||
USED_BY_NESTED_FRIEND2(std::vector< std::pair<int, FileMetaData> > new_files_)
|
||||
std::vector< std::pair<int, InternalKey> > compact_pointers_;
|
||||
DeletedFileSet deleted_files_;
|
||||
std::vector< std::pair<int, FileMetaData> > new_files_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
Loaded 100 of 291 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user