Compare commits

..
Author SHA1 Message Date
Brannon King 1069eb65b5 switched to use basho LevelDB
This is much more efficient in disk writes.
more careful incorporation of basho


made it build out of the box (but not well)
2019-08-20 09:50:49 -06:00
Brannon King 83319b7f31 optimized a little 2019-08-02 17:27:28 -06:00
Brannon King 6b8935718e first pass at not loading full claimtrie into RAM
tweaks
2019-08-01 18:35:58 -06:00
Brannon King 05d89e91cf fix unit test crash on OSX
pulled in some fixes from v18
2019-08-01 15:44:05 -06:00
291 changed files with 27274 additions and 17462 deletions

No files matched your search

+4 -4
View File
@@ -1,8 +1,8 @@
<!-- This issue tracker is only for technical issues related to lbrycrd (the LBRY blockchain).
<!-- This issue tracker is only for technical issues related to Bitcoin Core.
General questions and/or support requests are best directed to the community chat at https://chat.lbry.org.
General bitcoin questions and/or support requests are best directed to the Bitcoin StackExchange at https://bitcoin.stackexchange.com.
For reporting security issues, please email security@lbry.com.
For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/.
If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! -->
@@ -13,7 +13,7 @@ If the node is "stuck" during sync or giving "block checksum mismatch" errors, p
<!--- How reliably can you reproduce the issue, what are the steps to do so? -->
<!-- What version of lbrycrd are you using, where did you get it (website, self-compiled, etc)? -->
<!-- What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? -->
<!-- What type of machine are you observing the error on (OS/CPU and disk type)? -->
+18 -9
View File
@@ -7,6 +7,7 @@ cache:
stages:
- build
- test
- quality
jobs:
include:
@@ -14,7 +15,7 @@ jobs:
- &build-template
stage: build
name: linux
env: NAME=linux DOCKER_IMAGE=lbry/build_lbrycrd_gcc EXT=
env: NAME=linux EXT=
os: linux
dist: xenial
language: minimal
@@ -22,9 +23,8 @@ jobs:
- docker
install:
- mkdir -p ${HOME}/ccache
- docker pull $DOCKER_IMAGE
- docker pull $DOCKER_BUILD_IMAGE
script:
- echo "build..."
- docker run -v "$(pwd):/lbrycrd" -v "${HOME}/ccache:/ccache" -w /lbrycrd -e CCACHE_DIR=/ccache ${DOCKER_IMAGE} packaging/build_${NAME}_64bit.sh
before_deploy:
- mkdir -p dist
@@ -48,11 +48,11 @@ jobs:
- <<: *build-template
name: windows
env: NAME=windows DOCKER_IMAGE=lbry/build_lbrycrd EXT=.exe
env: NAME=windows EXT=.exe
- <<: *build-template
name: osx
env: NAME=darwin DOCKER_IMAGE=lbry/build_lbrycrd EXT=
env: NAME=darwin EXT=
before_install:
- mkdir -p ./depends/SDKs && pushd depends/SDKs && curl -C - ${MAC_OS_SDK} | tar --skip-old-files -xJ && popd
@@ -63,12 +63,12 @@ jobs:
dist: xenial
language: minimal
git:
clone: false
depth: 3
install:
- mkdir -p testrun && cd testrun
- curl http://build.lbry.io/lbrycrd/${TRAVIS_BRANCH}/lbrycrd-${NAME}-test.zip -o temp.zip
- unzip temp.zip
script: TRIEHASH_FUZZER_BLOCKS=1000 ./test_lbrycrd
script: ./test_lbrycrd
- <<: *test-template
# os: windows # doesn't support secrets at the moment
@@ -78,11 +78,20 @@ jobs:
services:
- docker
script:
- docker pull lbry/wine
- docker run -v "$(pwd):/test" -e "WINEDEBUG=-all" -e "TRIEHASH_FUZZER_BLOCKS=1000" -it lbry/wine wine "/test/test_lbrycrd.exe"
- docker pull $DOCKER_WINE_IMAGE
- docker run -v "$(pwd):/test" -e "WINEDEBUG=-all" -it $DOCKER_WINE_IMAGE wine "/test/test_lbrycrd.exe"
- <<: *test-template
os: osx
osx_image: xcode8.3
env: NAME=darwin
- stage: quality
name: "check format"
os: linux
dist: xenial
language: minimal
install:
- sudo apt-get install -y clang-format-3.9
script: git diff -U0 origin/master -- '*.h' '*.cpp' | ./contrib/devtools/clang-format-diff.py -p1
+34 -79
View File
@@ -1,35 +1,14 @@
# LBRYcrd - The LBRY blockchain
[![Build Status](https://travis-ci.org/lbryio/lbrycrd.svg?branch=master)](https://travis-ci.org/lbryio/lbrycrd)
[![MIT licensed](https://img.shields.io/dub/l/vibe-d.svg?style=flat)](https://github.com/lbryio/lbry-desktop/blob/master/LICENSE)
LBRYcrd uses a blockchain similar to bitcoin's to implement an index and payment system for content on the LBRY network. It is a fork of [bitcoin core](https://github.com/bitcoin/bitcoin). In addition to the libraries used by bitcoin, LBRYcrd also uses [icu4c](https://github.com/unicode-org/icu/tree/master/icu4c).
LBRYcrd uses a blockchain similar to bitcoin's to implement an index and payment system for content on the LBRY network. It is a fork of bitcoin core. In addition to the libraries used by bitcoin, LBRYcrd also uses icu4c.
Please read the [lbry.tech overview](https://lbry.tech/overview) for a general understanding of the LBRY pieces. From there you could read the [LBRY spec](https://spec.lbry.com/) for specifics on the data in the blockchain.
## Table of Contents
1. [Installation](#installation)
2. [Usage](#usage)
1. [Examples](#examples)
2. [Data directory](#data-directory)
3. [Running from Source](#running-from-source)
1. [Ubuntu with pulled static dependencies](#ubuntu-with-pulled-static-dependencies)
2. [Ubuntu with local shared dependencies](#ubuntu-with-local-shared-dependencies)
3. [MacOS (cross-compiled)](<#macos-(cross-compiled)>)
4. [MacOS with local shared dependencies](#macos-with-local-shared-dependencies)
5. [Windows (cross-compiled)](<#windows-(cross-compiled)>)
6. [Use with CLion](#use-with-clion)
4. [Contributing](#contributing)
- [Testnet](#testnet)
5. [Mailing List](#mailing-list)
6. [License](#license)
7. [Security](#security)
8. [Contact](#contact)
Please read the [lbry.tech overview](https://lbry.tech/overview) for a general understanding of the LBRY pieces. From there you could read the [LBRY spec](https://spec.lbry.com/) for specifics on the data in the blockchain.
## Installation
Latest binaries are available from https://github.com/lbryio/lbrycrd/releases. There is no installation procedure; the CLI binaries will run as-is and will have any uncommon dependencies statically linked into the binary. The QT GUI is not supported. LBRYcrd is distributed as a collection of executable files; traditional installers are not provided.
Latest binaries are available from https://github.com/lbryio/lbrycrd/releases. There is no installation procedure; the CLI binaries will run as-is and will have any uncommon dependencies statically linked into the binary. The QT GUI is not supported. LBRYcrd is distributed as a collection of executable files; traditional installers are not provided.
## Usage
@@ -37,7 +16,7 @@ The `lbrycrdd` executable will start a LBRYcrd node and connect you to the LBRYc
to interact with lbrycrdd through the command line. Command-line help for both executables are available through
the "--help" flag (e.g. `lbrycrdd --help`). Examples:
#### Examples
#### Examples:
Run `./lbrycrdd -server -daemon` to start lbrycrdd in the background.
@@ -46,8 +25,7 @@ Run `./lbrycrd-cli -getinfo` to check for some basic information about your LBRY
Run `./lbrycrd-cli help` to get a list of all commands that you can run. To get help on specific commands run `./lbrycrd-cli [command_name] help`
Test locally:
```sh
```
./lbrycrdd -server -regtest -txindex # run this in its own window
./lbrycrd-cli -regtest generate 120 # mine 20 spendable coins
./lbrycrd-cli -regtest claimname my_name deadbeef 1 # hold a name claim with 1 coin
@@ -57,26 +35,24 @@ Test locally:
./lbrycrd-cli -regtest stop # kill lbrycrdd
rm -fr ~/.lbrycrd/regtest/ # destroy regtest data
```
For further understanding of a "regtest" setup, see the local stack setup instructions here: https://lbry.tech/resources/regtest-setup
The CLI help is also browsable online at https://lbry.tech/api/blockchain
#### Data directory
#### Data directory:
Lbrycrdd will use the below default data directories (changeable with -datadir):
```sh
```
Windows: %APPDATA%\lbrycrd
Mac: ~/Library/Application Support/lbrycrd
Unix: ~/.lbrycrd
```
The data directory contains various things such as your default wallet (wallet.dat), debug logs (debug.log), and blockchain data. You can optionally create a configuration file lbrycrd.conf in the default data directory which will be used by default when running lbrycrdd.
For a list of configuration parameters, run `./lbrycrdd --help`. Below is a sample lbrycrd.conf to enable JSON RPC server on lbrycrdd.
```sh
```
rpcuser=lbry
rpcpassword=xyz123456790
daemon=1
@@ -85,19 +61,15 @@ txindex=1
```
## Running from Source
The easiest way to compile is to utilize the Docker image that contains the necessary compilers: lbry/build_lbrycrd. This will allow you to reproduce the build as made on our build servers. In this sample we map a local lbrycrd folder and a local ccache folder inside the image:
```sh
The easiest way to compile is to utilize the Docker image that contains the necessary compilers: lbry/build_lbrycrd. This will allow you to reproduce the build as made on our build servers. I this sample we map a local lbrycrd folder and a local ccache folder inside the image:
```
git clone https://github.com/lbryio/lbrycrd.git
cd lbrycrd
docker run -v "$(pwd):/lbrycrd" --rm -v "${HOME}/ccache:/ccache" -w /lbrycrd -e CCACHE_DIR=/ccache lbry/build_lbrycrd packaging/build_linux_64bit.sh
```
Some examples of compiling directly:
#### Ubuntu with pulled static dependencies
```sh
#### Ubuntu with pulled static dependencies:
```
sudo apt install build-essential git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates
git clone https://github.com/lbryio/lbrycrd.git
cd lbrycrd
@@ -105,20 +77,16 @@ cd lbrycrd
./src/test/test_lbrycrd
```
Other Linux distros would be similar. The build shell script is fairly trivial; take a peek at its contents.
#### Ubuntu with local shared dependencies
#### Ubuntu with local shared dependencies:
Note: using untested dependencies may lead to conflicting results.
```sh
```
sudo add-apt-repository ppa:bitcoin/bitcoin
sudo apt-get update
sudo apt-get install libdb4.8-dev libdb4.8++-dev libicu-dev libssl-dev libevent-dev \
sudo apt-get install libdb4.8-dev libdb4.8++-dev libicu-dev libssl-dev libevent-dev \
build-essential git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates \
libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev
# optionally include libminiupnpc-dev libzmq3-dev
git clone https://github.com/lbryio/lbrycrd.git
@@ -129,13 +97,11 @@ make -j$(nproc)
./src/lbrycrdd -server ...
```
#### MacOS (cross-compiled)
```sh
#### MacOS (cross-compiled):
```
sudo apt-get install clang llvm git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates \
libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev
git clone https://github.com/lbryio/lbrycrd.git
cd lbrycrd
# download MacOS SDK from your favorite source
@@ -144,12 +110,9 @@ tar ... extract SDK to depends/SDKs/MacOSX10.11.sdk
./packaging/build_darwin_64bit.sh
```
Look in packaging/build_darwin_64bit.sh for further understanding.
#### MacOS with local shared dependencies
```sh
#### MacOS with local shared dependencies:
```
brew install boost berkeley-db@4 icu4c libevent
# fix conflict with gawk pulled first:
brew reinstall readline
@@ -164,17 +127,14 @@ CONFIG_SITE=$(pwd)/depends/x86_64-apple-darwin15.6.0/share/config.site ./configu
make -j$(sysctl -n hw.ncpu)
```
#### Windows (cross-compiled)
#### Windows (cross-compiled):
Compiling on MS Windows (outside of WSL) is not supported. The Windows build is cross-compiled from Linux like so:
```sh
```
sudo apt-get install build-essential git libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates \
g++-mingw-w64-x86-64 mingw-w64-x86-64-dev
update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
git clone https://github.com/lbryio/lbrycrd.git
cd lbrycrd
./packaging/build_windows_64bit.sh
@@ -183,16 +143,13 @@ cd lbrycrd
If you encounter any errors, please check `doc/build-*.md` for further instructions. If you're still stuck, [create an issue](https://github.com/lbryio/lbrycrd/issues/new) with the output of that command, your system info, and any other information you think might be helpful. The scripts in the packaging folder are simple and will grant extra light on the build process as needed.
#### Use with CLion
#### Use with CLion:
CLion has not traditionally supported Autotools projects, although some progress on that is now in the works. We do include a cmake build file for compiling lbrycrd. See contrib/cmake. Alas, CLion doesn't support external projects in cmake, so that particular approach is also insufficient. CLion does support "compile_commands.json" projects. Fortunately, this can be easily generated for lbrycrd like so:
```sh
```
pip install --user compiledb
./autogen.sh && ./configure --enable-static=no --enable-shared --with-pic --without-gui CXXFLAGS="-O0 -g" CFLAGS="-O0 -g" # or whatever normal lbrycrd config
compiledb make -j10
```
Then open the newly generated compile_commands.json file as a project in CLion. Debugging is supported if you compiled with `-g`. To enable that you will need to create a target in CLion by going to File -> Settings -> Build -> Custom Build Targets. Add an empty target with your choice of name. From there you can go to "Edit Configurations", typically found in a drop-down at the top of the editor. Add a Custom Build Application, select your new target, select the compiled file (i.e. test_lbrycrd or lbrycrdd, etc), and then add any necessary command line parameters. Ensure that there is nothing in the "Before launch" section.
## Contributing
@@ -210,8 +167,8 @@ regularly to indicate new official, stable release versions.
Testing and code review is the bottleneck for development; we get more pull
requests than we can review and test on short notice. Please be patient and help out by testing
other people's pull requests, and remember this is a security-critical project where any mistake might cost people
lots of money. Developers are strongly encouraged to write [unit tests](/src/test/README.md) for new code and to
submit new unit tests for old code. Unit tests are compiled by default and can be run with `src/test/test_lbrycrd`
lots of money. Developers are strongly encouraged to write [unit tests](/doc/unit-tests.md) for new code and to
submit new unit tests for old code. Unit tests are compiled by default and can be run with `src/test/test_lbrycrd`.
The Travis CI system makes sure that every pull request is built, and that unit and sanity tests are automatically run. See https://travis-ci.org/lbryio/lbrycrd
@@ -219,11 +176,7 @@ The Travis CI system makes sure that every pull request is built, and that unit
Testnet is maintained for testing purposes and can be accessed using the command `./lbrycrdd -testnet`. If you would like to obtain testnet credits, please contact brannon@lbry.com or grin@lbry.com .
It is easy to solo mine on testnet. (It's easy on mainnet too, but much harder to win.) For instructions see [SGMiner](https://github.com/lbryio/sgminer-gm) and [Mining Contributions](https://github.com/lbryio/lbrycrd/tree/master/contrib/mining)
## Mailing List
We maintain a mailing list for notifications of upgrades, security issues, and soft/hard forks. To join, visit [https://lbry.com/forklist](https://lbry.com/forklist).
It is easy to solo mine on testnet. (It's easy on mainnet too, but much harder to win.) For instructions see https://github.com/lbryio/sgminer-gm and https://github.com/lbryio/lbrycrd/tree/master/contrib/mining
## License
@@ -231,9 +184,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact [security@lbry.com](mailto:security@lbry.com) regarding any security issues.
Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues.
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## Contact
The primary contact for this project is [@BrannonKing](https://github.com/BrannonKing) (brannon@lbry.com)
+1 -1
View File
@@ -7,7 +7,7 @@ export LC_ALL=C
set -e
srcdir="$(dirname $0)"
cd "$srcdir"
if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="$(which glibtoolize 2>/dev/null)"; then
if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="`which glibtoolize 2>/dev/null`"; then
LIBTOOLIZE="${GLIBTOOLIZE}"
export LIBTOOLIZE
fi
+3 -3
View File
@@ -2,10 +2,10 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N)
AC_PREREQ([2.60])
define(_CLIENT_VERSION_MAJOR, 0)
define(_CLIENT_VERSION_MINOR, 17)
define(_CLIENT_VERSION_REVISION, 3)
define(_CLIENT_VERSION_BUILD, 3)
define(_CLIENT_VERSION_REVISION, 2)
define(_CLIENT_VERSION_BUILD, 1)
define(_CLIENT_VERSION_IS_RELEASE, true)
define(_COPYRIGHT_YEAR, 2021)
define(_COPYRIGHT_YEAR, 2019)
define(_COPYRIGHT_HOLDERS,[The %s developers])
define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[LBRYcrd Core]])
AC_INIT([LBRYcrd Core],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION],[https://github.com/lbryio/lbrycrd/issues],[lbrycrd],[https://lbry.com/])
+3 -3
View File
@@ -36,13 +36,13 @@ if [ -z "${CODESIGN_ALLOCATE}" ]; then
fi
find ${TEMPDIR} -name "*.sign" | while read i; do
SIZE=$(stat -c %s "${i}")
TARGET_FILE="$(echo "${i}" | sed 's/\.sign$//')"
SIZE=`stat -c %s "${i}"`
TARGET_FILE="`echo "${i}" | sed 's/\.sign$//'`"
echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}"
${CODESIGN_ALLOCATE} -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp"
OFFSET=$(${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g')
OFFSET=`${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
if [ -z ${QUIET} ]; then
echo "Attaching signature at offset ${OFFSET}"
fi
+5 -5
View File
@@ -27,19 +27,19 @@ ${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}"
grep -v CodeResources < "${TEMPLIST}" | while read i; do
TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`"
SIZE=$(pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g')
OFFSET=$(pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g')
SIZE=`pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g'`
OFFSET=`pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'`
SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign"
DIRNAME="$(dirname "${SIGNFILE}")"
DIRNAME="`dirname "${SIGNFILE}"`"
mkdir -p "${DIRNAME}"
echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}"
dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null
done
grep CodeResources < "${TEMPLIST}" | while read i; do
TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")"
TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`"
RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}"
DIRNAME="$(dirname "${RESOURCE}")"
DIRNAME="`dirname "${RESOURCE}"`"
mkdir -p "${DIRNAME}"
echo "Adding resource for: \"${TARGETFILE}\""
cp "${i}" "${RESOURCE}"
+2 -3
View File
@@ -36,7 +36,7 @@ insert into coins(name, symbol, symbol2, algo, enable, auto_ready, rpcuser, rpcp
values('Local LBRY Instance', 'LBC', 'LBC', 'lbry', 1, 1, 'ruser', 'rpswd', '127.0.0.1', 19245, 1, 'utf-8', 0, 1, 0, 0, 0);
exit
```
Use port 19245 for testnet, port 9245 for main. Set usesegwit to 1 after the segwit fork is enabled on December 11, 2019.
Use port 19245 for testnet, port 9245 for main.
#### 3. Run the stratum server:
```
docker run --network host -d lbry/yiimp_stratum
@@ -47,11 +47,10 @@ docker run --network host -it lbry/yiimp_stratum bash
cat config/lbry.conf
./stratum config/lbry
```
When testing with an ASIC you may need to modify the TCP server address in said lbry.conf file to be an external IP address.
#### 4. Connect sgminer to it:
```
sgminer -k lbry -o stratum+tcp://127.0.0.1:3334/ -D -T -O mn824Su1wX7ip8WcNYzXwwWqvBvkeWGRo6:x
```
The username there is the account to receive payments from the pool. The password is unused. Tested with https://github.com/lbryio/sgminer-gm.
You can use whatever miner you prefer.
You can use whatever miner you prefer.
+1 -1
View File
@@ -23,7 +23,7 @@ TIMESERVER=http://timestamp.comodoca.com
CERTFILE="win-codesign.cert"
mkdir -p "${OUTSUBDIR}"
basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do
basename -a `ls -1 "${SRCDIR}"/*-unsigned.exe` | while read UNSIGNED; do
echo Signing "${UNSIGNED}"
"${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@"
"${OSSLSIGNCODE}" extract-signature -pem -in "${WORKDIR}/${UNSIGNED}" -out "${OUTSUBDIR}/${UNSIGNED}.pem" && rm "${WORKDIR}/${UNSIGNED}"
+1 -1
View File
@@ -131,7 +131,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
-e 's|@build_os@|$(build_os)|' \
-e 's|@host_os@|$(host_os)|' \
-e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \
-e 's|@CXXFLAGS@|$(strip -pipe $(host_$(release_type)_CXXFLAGS))|' \
-e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \
-e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \
-e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
-e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
File diff suppressed because it is too large. Load diff
+5 -6
View File
@@ -7,13 +7,12 @@ darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroo
darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B $(host_prefix)/native/bin
darwin_CFLAGS=-pipe
darwin_CXXFLAGS=$(darwin_CFLAGS) -std=c++11
darwin_CXXFLAGS=$(darwin_CFLAGS)
darwin_release_CFLAGS=-O2 -g
darwin_release_CFLAGS=-O2
darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
darwin_debug_CFLAGS=-Og -g
darwin_debug_CXXFLAGS=-O0 -g
darwin_native_toolchain=native_cctools
darwin_debug_CFLAGS=-Og
darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
darwin_native_toolchain=native_cctools
+4 -7
View File
@@ -1,14 +1,11 @@
linux_CFLAGS=-pipe
linux_CXXFLAGS=$(linux_CFLAGS) -std=c++11
linux_CXXFLAGS=$(linux_CFLAGS)
linux_release_CFLAGS=-O3 -g
ifeq (1,$(shell ldd --version | head -1 | awk '{print $$NF < 2.28}'))
linux_release_CFLAGS+= -include $(BASEDIR)/glibc_version_header/force_link_glibc_2.19.h
endif
linux_release_CFLAGS=-O2
linux_release_CXXFLAGS=$(linux_release_CFLAGS)
linux_debug_CFLAGS=-O1 -g
linux_debug_CXXFLAGS=-O0 -g
linux_debug_CFLAGS=-Og
linux_debug_CXXFLAGS=$(linux_debug_CFLAGS)
linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
+4 -5
View File
@@ -1,11 +1,10 @@
mingw32_CFLAGS=-pipe
mingw32_CXXFLAGS=$(mingw32_CFLAGS) -std=c++11
mingw32_CXXFLAGS=$(mingw32_CFLAGS)
mingw32_release_CFLAGS=-O2 -g
mingw32_release_CFLAGS=-O2
mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS)
mingw32_debug_CFLAGS=-O1 -g
mingw32_debug_CXXFLAGS=-O0 -g
mingw32_debug_CFLAGS=-O1
mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS)
mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
+2 -3
View File
@@ -1,6 +1,6 @@
package=bdb
$(package)_version=4.8.30
$(package)_download_path=https://download.oracle.com/berkeley-db
$(package)_download_path=http://download.oracle.com/berkeley-db
$(package)_file_name=db-$($(package)_version).NC.tar.gz
$(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
$(package)_build_subdir=build_unix
@@ -9,7 +9,7 @@ define $(package)_set_vars
$(package)_config_opts=--disable-shared --enable-cxx --disable-replication
$(package)_config_opts_mingw32=--enable-mingw
$(package)_config_opts_linux=--with-pic
$(package)_cppflags_mingw32=-DUNICODE -D_UNICODE
$(package)_cxxflags=-std=c++11
endef
define $(package)_preprocess_cmds
@@ -29,4 +29,3 @@ endef
define $(package)_stage_cmds
$(MAKE) DESTDIR=$($(package)_staging_dir) install_lib install_include
endef
+1 -1
View File
@@ -1,6 +1,6 @@
package=boost
$(package)_version=1_69_0
$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/
$(package)_download_path=https://dl.bintray.com/boostorg/release/1.69.0/source/
$(package)_file_name=$(package)_$($(package)_version).tar.bz2
$(package)_sha256_hash=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406
$(package)_dependencies=icu
+1 -2
View File
@@ -22,6 +22,7 @@ define $(package)_preprocess_cmds
PKG_CONFIG_SYSROOT_DIR=/ \
PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig \
PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig \
sed -i.old 's/^GEN_DEPS.cc.*/& $(CXXFLAGS)/' source/config/mh-mingw* && \
mkdir -p build && cd build && \
../source/runConfigureICU Linux $($(package)_standard_opts) CXXFLAGS=-std=c++11 && \
$(MAKE) && cd ..
@@ -31,8 +32,6 @@ define $(package)_config_cmds
PKG_CONFIG_SYSROOT_DIR=/ \
PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig \
PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig \
sed -i.old 's|^GEN_DEPS.c=.*|& $($(package)_cflags)|' config/mh-mingw* && \
sed -i.old 's|^GEN_DEPS.cc=.*|& $($(package)_cxxflags)|' config/mh-mingw* && \
$($(package)_autoconf)
endef
-1
View File
@@ -27,5 +27,4 @@ define $(package)_stage_cmds
endef
define $(package)_postprocess_cmds
rm lib/*.la
endef
+1 -1
View File
@@ -1,6 +1,6 @@
package=miniupnpc
$(package)_version=2.0.20180203
$(package)_download_path=https://miniupnp.tuxfamily.org/files/
$(package)_download_path=http://miniupnp.free.fr/files
$(package)_file_name=$(package)-$($(package)_version).tar.gz
$(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7
+2 -1
View File
@@ -5,7 +5,7 @@ $(package)_file_name=$(package)-$($(package)_version).tar.gz
$(package)_sha256_hash=8f9faeaebad088e772f4ef5e38252d472be4d878c6b3a2718c10a4fcebe7a41c
define $(package)_set_vars
$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc) $($(package)_cflags) $($(package)_cppflags)"
$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
$(package)_config_opts+=no-camellia
$(package)_config_opts+=no-capieng
@@ -42,6 +42,7 @@ $(package)_config_opts+=no-weak-ssl-ciphers
$(package)_config_opts+=no-whirlpool
$(package)_config_opts+=no-zlib
$(package)_config_opts+=no-zlib-dynamic
$(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags)
$(package)_config_opts_linux=-fPIC -Wa,--noexecstack
$(package)_config_opts_x86_64_linux=linux-x86_64
$(package)_config_opts_i686_linux=linux-generic32
+1
View File
@@ -4,6 +4,7 @@ $(package)_download_path=$(native_$(package)_download_path)
$(package)_file_name=$(native_$(package)_file_name)
$(package)_sha256_hash=$(native_$(package)_sha256_hash)
$(package)_dependencies=native_$(package)
$(package)_cxxflags=-std=c++11
define $(package)_set_vars
$(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc
+3 -4
View File
@@ -6,10 +6,9 @@ $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d83
$(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch
define $(package)_set_vars
$(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror --disable-drafts
$(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci
$(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov
$(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror
$(package)_config_opts_linux=--with-pic
$(package)_cxxflags=-std=c++11
endef
define $(package)_preprocess_cmds
@@ -32,5 +31,5 @@ endef
define $(package)_postprocess_cmds
sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \
rm -rf bin share lib/*.la
rm -rf bin share
endef
+3 -3
View File
@@ -30,13 +30,13 @@ if which ccache >/dev/null; then
fi
pushd depends
make -j$(getconf _NPROCESSORS_ONLN) HOST=x86_64-apple-darwin14 NO_QT=1 V=1
make -j`getconf _NPROCESSORS_ONLN` HOST=x86_64-apple-darwin14 NO_QT=1 V=1
popd
./autogen.sh
DEPS_DIR=$(pwd)/depends/x86_64-apple-darwin14
DEPS_DIR=`pwd`/depends/x86_64-apple-darwin14
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --enable-reduce-exports --without-gui --with-icu="${DEPS_DIR}" --enable-static --disable-shared
make -j$(getconf _NPROCESSORS_ONLN)
make -j`getconf _NPROCESSORS_ONLN`
${DEPS_DIR}/native/bin/x86_64-apple-darwin14-strip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx
if which ccache >/dev/null; then
+4 -4
View File
@@ -20,13 +20,13 @@ export CXXFLAGS="${CXXFLAGS:--frecord-gcc-switches}"
echo "CXXFLAGS set to $CXXFLAGS"
cd depends
make -j$(getconf _NPROCESSORS_ONLN) HOST=x86_64-pc-linux-gnu NO_QT=1 V=1
make -j`getconf _NPROCESSORS_ONLN` HOST=x86_64-pc-linux-gnu NO_QT=1 V=1
cd ..
./autogen.sh
DEPS_DIR=$(pwd)/depends/x86_64-pc-linux-gnu
DEPS_DIR=`pwd`/depends/x86_64-pc-linux-gnu
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --enable-static --disable-shared --with-pic --without-gui
make -j$(getconf _NPROCESSORS_ONLN)
make -j`getconf _NPROCESSORS_ONLN`
strip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx
if which ccache >/dev/null; then
@@ -34,4 +34,4 @@ if which ccache >/dev/null; then
ccache -s
fi
echo "Linux 64bit build is complete"
echo "Linux 64bit build is complete"
+6 -3
View File
@@ -20,14 +20,17 @@ if which ccache >/dev/null; then
ccache -ps
fi
export CXXFLAGS="${CXXFLAGS:--frecord-gcc-switches}"
echo "CXXFLAGS set to $CXXFLAGS"
pushd depends
make -j$(getconf _NPROCESSORS_ONLN) HOST=i686-w64-mingw32 NO_QT=1 V=1
make -j`getconf _NPROCESSORS_ONLN` HOST=i686-w64-mingw32 NO_QT=1 V=1
popd
./autogen.sh
DEPS_DIR=$(pwd)/depends/i686-w64-mingw32
DEPS_DIR=`pwd`/depends/i686-w64-mingw32
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --prefix=/ --without-gui --with-icu="$DEPS_DIR" --enable-static --disable-shared
make -j$(getconf _NPROCESSORS_ONLN)
make -j`getconf _NPROCESSORS_ONLN`
i686-w64-mingw32-strip src/lbrycrdd.exe src/lbrycrd-cli.exe src/lbrycrd-tx.exe
if which ccache >/dev/null; then
+6 -3
View File
@@ -19,14 +19,17 @@ if which ccache >/dev/null; then
ccache -ps
fi
export CXXFLAGS="${CXXFLAGS:--frecord-gcc-switches}"
echo "CXXFLAGS set to $CXXFLAGS"
pushd depends
make -j$(getconf _NPROCESSORS_ONLN) HOST=x86_64-w64-mingw32 NO_QT=1 V=1
make -j`getconf _NPROCESSORS_ONLN` HOST=x86_64-w64-mingw32 NO_QT=1 V=1
popd
./autogen.sh
DEPS_DIR=$(pwd)/depends/x86_64-w64-mingw32
DEPS_DIR=`pwd`/depends/x86_64-w64-mingw32
CONFIG_SITE=${DEPS_DIR}/share/config.site ./configure --prefix=/ --without-gui --with-icu="$DEPS_DIR" --enable-static --disable-shared
make -j$(getconf _NPROCESSORS_ONLN)
make -j`getconf _NPROCESSORS_ONLN`
x86_64-w64-mingw32-strip src/lbrycrdd.exe src/lbrycrd-cli.exe src/lbrycrd-tx.exe
if which ccache >/dev/null; then
+1 -3
View File
@@ -155,7 +155,6 @@ BITCOIN_CORE_H = \
reverse_iterator.h \
reverselock.h \
rpc/blockchain.h \
rpc/claimrpchelp.h \
rpc/client.h \
rpc/mining.h \
rpc/protocol.h \
@@ -476,7 +475,6 @@ lbrycrdd_LDADD = \
$(LIBBITCOIN_CONSENSUS) \
$(LIBBITCOIN_CRYPTO) \
$(LIBLEVELDB) \
$(LIBLEVELDB_SSE42) \
$(LIBMEMENV) \
$(LIBSECP256K1)
@@ -574,7 +572,7 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps)
clean-local:
-$(MAKE) -C secp256k1 clean
-$(MAKE) -C univalue clean
-rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno
-$(MAKE) -C leveldb clean
-rm -f config.h
-rm -rf test/__pycache__
-1
View File
@@ -42,7 +42,6 @@ bench_bench_bitcoin_LDADD = \
$(LIBBITCOIN_CONSENSUS) \
$(LIBBITCOIN_CRYPTO) \
$(LIBLEVELDB) \
$(LIBLEVELDB_SSE42) \
$(LIBMEMENV) \
$(LIBSECP256K1) \
$(LIBUNIVALUE)
+7 -132
View File
@@ -2,148 +2,23 @@
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
SUBDIRS = leveldb
LIBLEVELDB_INT = leveldb/libleveldb.a
LIBMEMENV_INT = leveldb/libmemenv.a
LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a
EXTRA_LIBRARIES += $(LIBLEVELDB_INT)
EXTRA_LIBRARIES += $(LIBMEMENV_INT)
EXTRA_LIBRARIES += $(LIBLEVELDB_SSE42_INT)
LIBLEVELDB += $(LIBLEVELDB_INT)
LIBMEMENV += $(LIBMEMENV_INT)
LIBLEVELDB_SSE42 = $(LIBLEVELDB_SSE42_INT)
LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include
LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv
LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb
LEVELDB_CPPFLAGS_INT =
LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb
LEVELDB_CPPFLAGS_INT += $(LEVELDB_TARGET_FLAGS)
LEVELDB_CPPFLAGS_INT += -DLEVELDB_ATOMIC_PRESENT
LEVELDB_CPPFLAGS_INT += -D__STDC_LIMIT_MACROS
leveldb/libleveldb.a:
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C leveldb
if TARGET_WINDOWS
LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1
else
LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX
endif
leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS)
leveldb_libleveldb_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
leveldb_libleveldb_a_SOURCES=
leveldb_libleveldb_a_SOURCES += leveldb/port/atomic_pointer.h
leveldb_libleveldb_a_SOURCES += leveldb/port/port_example.h
leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.h
leveldb_libleveldb_a_SOURCES += leveldb/port/win/stdint.h
leveldb_libleveldb_a_SOURCES += leveldb/port/port.h
leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.h
leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h
leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h
leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h
leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h
leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h
leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h
leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h
leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h
leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h
leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h
leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h
leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h
leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h
leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h
leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h
leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h
leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h
leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h
leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h
leveldb_libleveldb_a_SOURCES += leveldb/table/block.h
leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h
leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h
leveldb_libleveldb_a_SOURCES += leveldb/table/format.h
leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h
leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h
leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h
leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h
leveldb_libleveldb_a_SOURCES += leveldb/util/random.h
leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h
leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h
leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h
leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h
leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h
leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h
leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h
leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h
leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc
leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc
leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc
leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc
if TARGET_WINDOWS
leveldb_libleveldb_a_SOURCES += leveldb/util/env_win.cc
leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.cc
else
leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.cc
endif
leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc
leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h
leveldb_libleveldb_sse42_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS)
leveldb_libleveldb_sse42_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS)
if ENABLE_HWCRC32
leveldb_libleveldb_sse42_a_CPPFLAGS += -DLEVELDB_PLATFORM_POSIX_SSE
leveldb_libleveldb_sse42_a_CXXFLAGS += $(SSE42_CXXFLAGS)
endif
leveldb_libleveldb_sse42_a_SOURCES = leveldb/port/port_posix_sse.cc
leveldb/libmemenv.a: leveldb/libleveldb.a
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C leveldb memenv_test
+1 -1
View File
@@ -408,7 +408,7 @@ endif
if ENABLE_ZMQ
qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \
qt_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \
$(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
qt_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
+1 -1
View File
@@ -63,7 +63,7 @@ if ENABLE_ZMQ
qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
qt_test_test_lbrycrd_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \
$(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
$(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \
$(QR_LIBS) $(PROTOBUF_LIBS) $(ICU_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \
$(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
qt_test_test_lbrycrd_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
+1 -6
View File
@@ -66,11 +66,6 @@ BITCOIN_TESTS =\
test/net_tests.cpp \
test/claimtriecache_tests.cpp \
test/claimtriebranching_tests.cpp \
test/claimtrieexpirationfork_tests.cpp \
test/claimtriefixture.cpp \
test/claimtriehashfork_tests.cpp \
test/claimtrienormalization_tests.cpp \
test/claimtrierpc_tests.cpp \
test/nameclaim_tests.cpp \
test/netbase_tests.cpp \
test/pmt_tests.cpp \
@@ -127,7 +122,7 @@ test_test_lbrycrd_LDADD += $(LIBBITCOIN_WALLET)
endif
test_test_lbrycrd_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
$(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
$(LIBLEVELDB) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
test_test_lbrycrd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
test_test_lbrycrd_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(CRYPTO_LIBS) $(ICU_LIBS) $(MINIUPNPC_LIBS)
-1
View File
@@ -3,7 +3,6 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <bloom.h>
#include <nameclaim.h>
#include <primitives/transaction.h>
#include <hash.h>
+15 -20
View File
@@ -123,7 +123,7 @@ void CChainParams::UpdateVersionBitsParameters(Consensus::DeploymentPos d, int64
class CMainParams : public CChainParams {
public:
CMainParams() {
strNetworkID = CBaseChainParams::MAIN;
strNetworkID = "lbrycrd";
consensus.nSubsidyLevelInterval = 1<<5;
consensus.nMajorityEnforceBlockUpgrade = 750;
consensus.nMajorityRejectBlockOutdated = 950;
@@ -144,9 +144,7 @@ public:
consensus.nAllowMinDiffMaxHeight = -1;
consensus.nNormalizedNameForkHeight = 539940; // targeting 21 March 2019
consensus.nMinTakeoverWorkaroundHeight = 496850;
consensus.nMaxTakeoverWorkaroundHeight = 658300; // targeting 30 Oct 2019
consensus.nWitnessForkHeight = 680770; // targeting 11 Dec 2019
consensus.nAllClaimsInMerkleForkHeight = 658310; // targeting 30 Oct 2019
consensus.nMaxTakeoverWorkaroundHeight = 10000000;
consensus.fPowAllowMinDifficultyBlocks = false;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 1916; // 95% of a half week
@@ -160,16 +158,17 @@ public:
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 1462060800; // May 1st, 2016
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 1493596800; // May 1st, 2017
// Deployment of SegWit (BIP141, BIP143, and BIP147) -- Unused (see nWitnessForkHeight).
// Deployment of SegWit (BIP141, BIP143, and BIP147)
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1;
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1547942400; // Jan 20, 2019
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1548288000; // Jan 24, 2019
// The best chain should have at least this much work.
consensus.nMinimumChainWork = uint256S("000000000000000000000000000000000000000000000499ed6684d1bf6f6fd3"); //946000
consensus.nMinimumChainWork = uint256S("0x000000000000000000000000000000000000000000000000607ca7e806c4c1e9"); //400000
// By default assume that the signatures in ancestors of this block are valid.
consensus.defaultAssumeValid = uint256S("0d3b537afe49820e1c6efc555463f955251b1293c6e5130137e1e25744431172"); //946000
//consensus.defaultAssumeValid = uint256S("0xf0e56e70782af63ccb49c76e852540688755869ba59ec68cac9c04a6b4d9f5ca"); //400000
consensus.defaultAssumeValid = uint256S("0xa6bbb48f5343eb9b0287c22f3ea8b29f36cf10794a37f8a925a894d6f4519913"); //4000
/**
* The message start string is designed to be unlikely to occur in normal data.
@@ -195,11 +194,9 @@ public:
vSeeds.clear();
vFixedSeeds.clear();
vSeeds.emplace_back("dnsseed1.lbry.io"); // LBRY Inc
vSeeds.emplace_back("dnsseed2.lbry.io"); // LBRY Inc
vSeeds.emplace_back("dnsseed3.lbry.io"); // LBRY Inc
vSeeds.emplace_back("seed.lbry.grin.io"); // Grin
vSeeds.emplace_back("seed.allaboutlbc.com"); // Madiator2011
vSeeds.emplace_back("dnsseed1.lbry.io"); // lbry.io
vSeeds.emplace_back("dnsseed2.lbry.io"); // lbry.io
vSeeds.emplace_back("dnsseed3.lbry.io"); // lbry.io
base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1, 0x55);
base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1, 0x7a);
@@ -207,6 +204,8 @@ public:
base58Prefixes[EXT_PUBLIC_KEY] = {0x04, 0x88, 0xB2, 0x1E};
base58Prefixes[EXT_SECRET_KEY] = {0x04, 0x88, 0xAD, 0xE4};
vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
bech32_hrp = "lbc";
vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
@@ -241,7 +240,7 @@ public:
class CTestNetParams : public CChainParams {
public:
CTestNetParams() {
strNetworkID = CBaseChainParams::TESTNET;
strNetworkID = "lbrycrdtest";
consensus.nSubsidyLevelInterval = 1 << 5;
consensus.nMajorityEnforceBlockUpgrade = 51;
consensus.nMajorityRejectBlockOutdated = 75;
@@ -262,9 +261,7 @@ public:
consensus.nAllowMinDiffMaxHeight = 1100000;
consensus.nNormalizedNameForkHeight = 993380; // targeting, 21 Feb 2019
consensus.nMinTakeoverWorkaroundHeight = 99;
consensus.nMaxTakeoverWorkaroundHeight = 1198550; // targeting 30 Sep 2019
consensus.nWitnessForkHeight = 1198600;
consensus.nAllClaimsInMerkleForkHeight = 1198560; // targeting 30 Sep 2019
consensus.nMaxTakeoverWorkaroundHeight = 10000000;
consensus.fPowAllowMinDifficultyBlocks = true;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 1512; // 75% for testchains
@@ -278,7 +275,7 @@ public:
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 1456790400; // March 1st, 2016
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 1493596800; // May 1st, 2017
// Deployment of SegWit (BIP141, BIP143, and BIP147) -- Unused (see nWitnessForkHeight).
// Deployment of SegWit (BIP141, BIP143, and BIP147)
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1;
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1462060800; // May 1st 2016
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1493596800; // May 1st 2017
@@ -351,7 +348,7 @@ public:
class CRegTestParams : public CChainParams {
public:
CRegTestParams() {
strNetworkID = CBaseChainParams::REGTEST;
strNetworkID = "lbrycrdreg";
consensus.nSubsidyLevelInterval = 1 << 5;
consensus.BIP16Exception = uint256();
consensus.BIP34Height = 1000; // BIP34 is needed for validation_block_tests
@@ -370,8 +367,6 @@ public:
consensus.nNormalizedNameForkHeight = 250; // SDK depends upon this number
consensus.nMinTakeoverWorkaroundHeight = -1;
consensus.nMaxTakeoverWorkaroundHeight = -1;
consensus.nWitnessForkHeight = 150;
consensus.nAllClaimsInMerkleForkHeight = 350;
consensus.fPowAllowMinDifficultyBlocks = false;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 108; // 75% for testchains
+1 -1
View File
@@ -13,7 +13,7 @@
const std::string CBaseChainParams::MAIN = "lbrycrd";
const std::string CBaseChainParams::TESTNET = "lbrycrdtest";
const std::string CBaseChainParams::REGTEST = "lbrycrdreg";
const std::string CBaseChainParams::REGTEST = "regtest";
void SetupChainParamsBaseOptions()
{
+1 -1
View File
@@ -149,7 +149,7 @@ bool ProcessClaim(CClaimScriptOp& claimOp, CClaimTrieCache& trieCache, const CSc
{
int op;
std::vector<std::vector<unsigned char> > vvchParams;
if (!DecodeClaimScript(scriptPubKey, op, vvchParams, trieCache.allowSupportMetadata()))
if (!DecodeClaimScript(scriptPubKey, op, vvchParams))
return false;
switch (op) {
+12 -12
View File
@@ -59,17 +59,17 @@ public:
*/
CClaimScriptAddOp(const COutPoint& point, CAmount nValue, int nHeight);
/**
* Implementation of OP_CLAIM_NAME handler
* Implamention of OP_CLAIM_NAME handler
* @see CClaimScriptOp::claimName
*/
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
/**
* Implementation of OP_UPDATE_CLAIM handler
* Implamention of OP_UPDATE_CLAIM handler
* @see CClaimScriptOp::updateClaim
*/
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
/**
* Implementation of OP_SUPPORT_CLAIM handler
* Implamention of OP_SUPPORT_CLAIM handler
* @see CClaimScriptOp::supportClaim
*/
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
@@ -100,17 +100,17 @@ public:
*/
CClaimScriptUndoAddOp(const COutPoint& point, int nHeight);
/**
* Implementation of OP_CLAIM_NAME handler
* Implamention of OP_CLAIM_NAME handler
* @see CClaimScriptOp::claimName
*/
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
/**
* Implementation of OP_UPDATE_CLAIM handler
* Implamention of OP_UPDATE_CLAIM handler
* @see CClaimScriptOp::updateClaim
*/
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
/**
* Implementation of OP_SUPPORT_CLAIM handler
* Implamention of OP_SUPPORT_CLAIM handler
* @see CClaimScriptOp::supportClaim
*/
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
@@ -141,17 +141,17 @@ public:
*/
CClaimScriptSpendOp(const COutPoint& point, int nHeight, int& nValidHeight);
/**
* Implementation of OP_CLAIM_NAME handler
* Implamention of OP_CLAIM_NAME handler
* @see CClaimScriptOp::claimName
*/
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
/**
* Implementation of OP_UPDATE_CLAIM handler
* Implamention of OP_UPDATE_CLAIM handler
* @see CClaimScriptOp::updateClaim
*/
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
/**
* Implementation of OP_SUPPORT_CLAIM handler
* Implamention of OP_SUPPORT_CLAIM handler
* @see CClaimScriptOp::supportClaim
*/
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
@@ -184,17 +184,17 @@ public:
*/
CClaimScriptUndoSpendOp(const COutPoint& point, CAmount nValue, int nHeight, int nValidHeight);
/**
* Implementation of OP_CLAIM_NAME handler
* Implamention of OP_CLAIM_NAME handler
* @see CClaimScriptOp::claimName
*/
bool claimName(CClaimTrieCache& trieCache, const std::string& name) override;
/**
* Implementation of OP_UPDATE_CLAIM handler
* Implamention of OP_UPDATE_CLAIM handler
* @see CClaimScriptOp::updateClaim
*/
bool updateClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
/**
* Implementation of OP_SUPPORT_CLAIM handler
* Implamention of OP_SUPPORT_CLAIM handler
* @see CClaimScriptOp::supportClaim
*/
bool supportClaim(CClaimTrieCache& trieCache, const std::string& name, const uint160& claimId) override;
+364 -358
View File
File diff suppressed because it is too large. Load diff
+96 -184
View File
@@ -18,8 +18,9 @@
#include <unordered_set>
// leveldb keys
#define TRIE_NODE 'n'
#define TRIE_NODE_CHILDREN 'b'
#define TRIE_NODE 'n' // deprecated
#define TRIE_NODE_BY_HASH 'h'
#define TRIE_NODE_BY_NAME 'g'
#define CLAIM_BY_ID 'i'
#define CLAIM_QUEUE_ROW 'r'
#define CLAIM_QUEUE_NAME_ROW 'm'
@@ -35,10 +36,10 @@ struct CClaimValue
{
COutPoint outPoint;
uint160 claimId;
CAmount nAmount = 0;
CAmount nEffectiveAmount = 0;
int nHeight = 0;
int nValidAtHeight = 0;
CAmount nAmount;
CAmount nEffectiveAmount;
int nHeight;
int nValidAtHeight;
CClaimValue() = default;
@@ -62,6 +63,7 @@ struct CClaimValue
READWRITE(nAmount);
READWRITE(nHeight);
READWRITE(nValidAtHeight);
READWRITE(nEffectiveAmount);
}
bool operator<(const CClaimValue& other) const
@@ -92,9 +94,9 @@ struct CSupportValue
{
COutPoint outPoint;
uint160 supportedClaimId;
CAmount nAmount = 0;
int nHeight = 0;
int nValidAtHeight = 0;
CAmount nAmount;
int nHeight;
int nValidAtHeight;
CSupportValue() = default;
@@ -158,17 +160,6 @@ struct CClaimTrieData
inline void SerializationOp(Stream& s, Operation ser_action)
{
READWRITE(hash);
if (ser_action.ForRead()) {
if (s.eof()) {
claims.clear();
nHeightOfLastTakeover = 0;
return;
}
}
else if (claims.empty())
return;
READWRITE(claims);
READWRITE(nHeightOfLastTakeover);
}
@@ -189,10 +180,34 @@ struct CClaimTrieData
}
};
struct CClaimTrieDataNode {
CClaimTrieData data;
// we're using a vector to avoid RAM thrashing and for faster serialization ops.
// We're assuming its data is inserted in order and never modified.
std::vector<std::pair<std::string, uint256>> children;
bool childrenSerialization = true;
CClaimTrieDataNode() = default;
CClaimTrieDataNode(CClaimTrieDataNode&&) = default;
CClaimTrieDataNode(const CClaimTrieDataNode&) = default;
CClaimTrieDataNode& operator=(CClaimTrieDataNode&&) = default;
CClaimTrieDataNode& operator=(const CClaimTrieDataNode& d) = default;
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action)
{
READWRITE(data);
if (childrenSerialization) // wanting constexpr but hoping the compiler is smart enough anyway
READWRITE(children);
}
};
struct COutPointHeightType
{
COutPoint outPoint;
int nHeight = 0;
int nHeight;
COutPointHeightType() = default;
@@ -215,7 +230,7 @@ struct CNameOutPointHeightType
{
std::string name;
COutPoint outPoint;
int nHeight = 0;
int nHeight;
CNameOutPointHeightType() = default;
@@ -284,73 +299,35 @@ struct CClaimIndexElement
CClaimValue claim;
};
struct CClaimNsupports
struct CClaimsForNameType
{
CClaimNsupports() = default;
CClaimNsupports(CClaimNsupports&&) = default;
CClaimNsupports(const CClaimNsupports&) = default;
claimEntryType claims;
supportEntryType supports;
int nLastTakeoverHeight;
std::string name;
CClaimNsupports& operator=(CClaimNsupports&&) = default;
CClaimNsupports& operator=(const CClaimNsupports&) = default;
CClaimNsupports(const CClaimValue& claim, CAmount effectiveAmount, const std::vector<CSupportValue>& supports = {})
: claim(claim), effectiveAmount(effectiveAmount), supports(supports)
CClaimsForNameType(claimEntryType claims, supportEntryType supports, int nLastTakeoverHeight, std::string name)
: claims(std::move(claims)), supports(std::move(supports)), nLastTakeoverHeight(nLastTakeoverHeight), name(std::move(name))
{
}
bool IsNull() const
{
return claim.claimId.IsNull();
}
CClaimValue claim;
CAmount effectiveAmount = 0;
std::vector<CSupportValue> supports;
CClaimsForNameType(CClaimsForNameType&&) = default;
CClaimsForNameType(const CClaimsForNameType&) = default;
CClaimsForNameType& operator=(CClaimsForNameType&&) = default;
CClaimsForNameType& operator=(const CClaimsForNameType&) = default;
};
static const CClaimNsupports invalid;
struct CClaimSupportToName
class CClaimTrie
{
CClaimSupportToName(const std::string& name, int nLastTakeoverHeight, std::vector<CClaimNsupports> claimsNsupports, std::vector<CSupportValue> unmatchedSupports)
: name(name), nLastTakeoverHeight(nLastTakeoverHeight), claimsNsupports(std::move(claimsNsupports)), unmatchedSupports(std::move(unmatchedSupports))
{
}
int nNextHeight = 0;
int nProportionalDelayFactor = 0;
std::unique_ptr<CDBWrapper> db;
const CClaimNsupports& find(const uint160& claimId) const
{
auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&claimId](const CClaimNsupports& value) {
return claimId == value.claim.claimId;
});
return it != claimsNsupports.end() ? *it : invalid;
}
const CClaimNsupports& find(const std::string& partialId) const
{
std::string lowered(partialId);
for (auto& c: lowered)
c = std::tolower(c);
auto it = std::find_if(claimsNsupports.begin(), claimsNsupports.end(), [&lowered](const CClaimNsupports& value) {
return value.claim.claimId.GetHex().find(lowered) == 0;
});
return it != claimsNsupports.end() ? *it : invalid;
}
const std::string name;
const int nLastTakeoverHeight;
const std::vector<CClaimNsupports> claimsNsupports;
const std::vector<CSupportValue> unmatchedSupports;
};
class CClaimTrie : public CPrefixTrie<std::string, CClaimTrieData>
{
public:
CClaimTrie() = default;
virtual ~CClaimTrie() = default;
CClaimTrie(CClaimTrie&&) = delete;
CClaimTrie(const CClaimTrie&) = delete;
CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor = 32, std::size_t cacheMB=200);
CClaimTrie(bool fMemory, bool fWipe, int proportionalDelayFactor = 32);
CClaimTrie& operator=(CClaimTrie&&) = delete;
CClaimTrie& operator=(const CClaimTrie&) = delete;
@@ -361,17 +338,19 @@ public:
friend struct ClaimTrieChainFixture;
friend class CClaimTrieCacheExpirationFork;
friend class CClaimTrieCacheNormalizationFork;
friend bool getClaimById(const uint160&, std::string&, CClaimValue*);
friend bool getClaimById(const std::string&, std::string&, CClaimValue*);
std::size_t getTotalNamesInTrie() const;
std::size_t getTotalClaimsInTrie() const;
CAmount getTotalValueOfClaimsInTrie(bool fControllingOnly) const;
bool checkConsistency(const uint256& rootHash) const;
protected:
int nNextHeight = 0;
int nProportionalDelayFactor = 0;
std::unique_ptr<CDBWrapper> db;
bool contains(const std::string& key) const;
bool empty() const;
bool find(const uint256& key, CClaimTrieDataNode& node) const;
bool find(const std::string& key, CClaimTrieDataNode& node) const;
std::vector<std::pair<std::string, CClaimTrieDataNode>> nodes(const std::string& key) const;
void recurseAllHashedNodes(const std::string& name, const CClaimTrieDataNode& current, std::function<void(const std::string&, const CClaimTrieDataNode&)> function) const;
};
struct CClaimTrieProofNode
@@ -394,66 +373,21 @@ struct CClaimTrieProofNode
struct CClaimTrieProof
{
CClaimTrieProof() = default;
CClaimTrieProof(std::vector<CClaimTrieProofNode> nodes, bool hasValue, const COutPoint& outPoint, int nHeightOfLastTakeover)
: nodes(std::move(nodes)), hasValue(hasValue), outPoint(outPoint), nHeightOfLastTakeover(nHeightOfLastTakeover)
{
}
CClaimTrieProof(CClaimTrieProof&&) = default;
CClaimTrieProof(const CClaimTrieProof&) = default;
CClaimTrieProof& operator=(CClaimTrieProof&&) = default;
CClaimTrieProof& operator=(const CClaimTrieProof&) = default;
std::vector<std::pair<bool, uint256>> pairs;
std::vector<CClaimTrieProofNode> nodes;
int nHeightOfLastTakeover = 0;
bool hasValue = false;
bool hasValue;
COutPoint outPoint;
};
template <typename T>
class COptional
{
bool own;
T* value;
public:
COptional(T* value = nullptr) : own(false), value(value) {}
COptional(COptional&& o)
{
own = o.own;
value = o.value;
o.own = false;
o.value = nullptr;
}
COptional(T&& o) : own(true)
{
value = new T(std::move(o));
}
~COptional()
{
if (own)
delete value;
}
COptional& operator=(COptional&&) = delete;
bool unique() const
{
return own;
}
operator bool() const
{
return value;
}
operator T*() const
{
return value;
}
T* operator->() const
{
return value;
}
operator T&() const
{
return *value;
}
T& operator*() const
{
return *value;
}
int nHeightOfLastTakeover;
};
template <typename T>
@@ -476,6 +410,8 @@ typedef std::map<int, expirationQueueRowType> expirationQueueType;
typedef std::set<CClaimValue> claimIndexClaimListType;
typedef std::vector<CClaimIndexElement> claimIndexElementListType;
typedef CPrefixTrie<std::string, CClaimTrieData> CClaimPrefixTrie;
class CClaimTrieCacheBase
{
public:
@@ -484,16 +420,17 @@ public:
uint256 getMerkleHash();
bool getClaimById(const uint160& claimId, std::string& name, CClaimValue& claim) const;
bool flush();
bool empty() const;
bool checkConsistency() const;
bool ReadFromDisk(const CBlockIndex* tip);
bool haveClaim(const std::string& name, const COutPoint& outPoint) const;
bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
bool haveClaimInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
bool haveSupport(const std::string& name, const COutPoint& outPoint) const;
bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
bool haveSupportInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
bool addClaim(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
bool undoAddClaim(const std::string& name, const COutPoint& outPoint, int nHeight);
@@ -523,24 +460,25 @@ public:
virtual int expirationTime() const;
virtual bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
virtual CClaimSupportToName getClaimsForName(const std::string& name) const;
virtual CClaimsForNameType getClaimsForName(const std::string& name) const;
CClaimTrie::const_iterator find(const std::string& name) const;
void iterate(std::function<void(const std::string&, const CClaimTrieData&)> callback) const;
CAmount getEffectiveAmountForClaim(const std::string& name, const uint160& claimId, std::vector<CSupportValue>* supports = nullptr) const;
CAmount getEffectiveAmountForClaim(const CClaimsForNameType& claims, const uint160& claimId, std::vector<CSupportValue>* supports = nullptr) const;
void dumpToLog(CClaimTrie::const_iterator it, bool diffFromBase = true) const;
CClaimPrefixTrie::const_iterator begin() const;
CClaimPrefixTrie::const_iterator end() const;
void dumpToLog(CClaimPrefixTrie::const_iterator it, bool diffFromBase = true) const;
virtual std::string adjustNameForValidHeight(const std::string& name, int validHeight) const;
protected:
CClaimTrie* base;
CClaimTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
CClaimPrefixTrie nodesToAddOrUpdate; // nodes pulled in from base (and possibly modified thereafter), written to base on flush
std::unordered_set<std::string> namesToCheckForTakeover; // takeover numbers are updated on increment
virtual uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it);
virtual bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const;
uint256 recursiveComputeMerkleHash(CClaimPrefixTrie::iterator& it);
virtual bool insertClaimIntoTrie(const std::string& name, const CClaimValue& claim, bool fCheckTakeover);
virtual bool removeClaimFromTrie(const std::string& name, const COutPoint& outPoint, CClaimValue& claim, bool fCheckTakeover);
@@ -553,7 +491,7 @@ protected:
int getDelayForName(const std::string& name) const;
virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
CClaimTrie::iterator cacheData(const std::string& name, bool create = true);
CClaimPrefixTrie::iterator cacheData(const std::string& name, bool create = true);
bool getLastTakeoverForName(const std::string& name, uint160& claimId, int& takeoverHeight) const;
@@ -582,8 +520,10 @@ private:
std::unordered_map<std::string, supportEntryType> supportCache; // to be added/updated to base (and disk) on flush
std::unordered_set<std::string> nodesToDelete; // to be removed from base (and disk) on flush
std::unordered_set<std::string> nodesAlreadyCached; // set of nodes already pulled into cache from base
std::unordered_map<std::string, bool> takeoverWorkaround;
std::unordered_set<std::string> removalWorkaround;
std::unordered_set<std::string> dirtyNodes;
bool shouldUseTakeoverWorkaround(const std::string& key) const;
void addTakeoverWorkaroundPotential(const std::string& key);
@@ -595,26 +535,19 @@ private:
bool removeSupport(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
bool removeClaim(const std::string& name, const COutPoint& outPoint, int nHeight, int& nValidAtHeight, bool fCheckTakeover);
template <typename T>
void insertRowsFromQueue(std::vector<T>& result, const std::string& name) const;
bool validateTrieConsistency(const CBlockIndex* tip);
template <typename T>
std::vector<queueEntryType<T>>* getQueueCacheRow(int nHeight, bool createIfNotExists);
std::pair<const int, std::vector<queueEntryType<T>>>* getQueueCacheRow(int nHeight, bool createIfNotExists = false);
template <typename T>
COptional<const std::vector<queueEntryType<T>>> getQueueCacheRow(int nHeight) const;
typename queueNameType::value_type* getQueueCacheNameRow(const std::string& name, bool createIfNotExists = false);
template <typename T>
queueNameRowType* getQueueCacheNameRow(const std::string& name, bool createIfNotExists);
typename expirationQueueType::value_type* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists = false);
template <typename T>
COptional<const queueNameRowType> getQueueCacheNameRow(const std::string& name) const;
template <typename T>
expirationQueueRowType* getExpirationQueueCacheRow(int nHeight, bool createIfNotExists);
template <typename T>
bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight) const;
bool haveInQueue(const std::string& name, const COutPoint& outPoint, int& nValidAtHeight);
template <typename T>
T add(const std::string& name, const COutPoint& outPoint, const uint160& claimId, CAmount nAmount, int nHeight);
@@ -662,8 +595,7 @@ public:
void setExpirationTime(int time);
int expirationTime() const override;
virtual void initializeIncrement();
bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
void expirationForkActive(int height, bool increment);
bool incrementBlock(insertUndoType& insertUndo,
claimQueueRowType& expireUndo,
@@ -708,7 +640,7 @@ public:
bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
bool getInfoForName(const std::string& name, CClaimValue& claim) const override;
CClaimSupportToName getClaimsForName(const std::string& name) const override;
CClaimsForNameType getClaimsForName(const std::string& name) const override;
std::string adjustNameForValidHeight(const std::string& name, int validHeight) const override;
protected:
@@ -731,26 +663,6 @@ private:
std::vector<std::pair<std::string, int>>& takeoverHeightUndo);
};
class CClaimTrieCacheHashFork : public CClaimTrieCacheNormalizationFork
{
public:
explicit CClaimTrieCacheHashFork(CClaimTrie* base);
bool getProofForName(const std::string& name, CClaimTrieProof& proof) override;
bool getProofForName(const std::string& name, CClaimTrieProof& proof, const std::function<bool(const CClaimValue&)>& comp);
void initializeIncrement() override;
bool finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo) override;
bool allowSupportMetadata() const;
protected:
uint256 recursiveComputeMerkleHash(CClaimTrie::iterator& it) override;
bool recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const override;
private:
void copyAllBaseToCache();
};
typedef CClaimTrieCacheHashFork CClaimTrieCache;
typedef CClaimTrieCacheNormalizationFork CClaimTrieCache;
#endif // BITCOIN_CLAIMTRIE_H
+15 -247
View File
@@ -1,9 +1,9 @@
#include <consensus/merkle.h>
#include <chainparams.h>
#include <claimtrie.h>
#include <hash.h>
#include <boost/algorithm/string.hpp>
#include <boost/foreach.hpp>
#include <boost/locale.hpp>
#include <boost/locale/conversion.hpp>
#include <boost/locale/localization_backend.hpp>
@@ -44,21 +44,10 @@ bool CClaimTrieCacheExpirationFork::decrementBlock(insertUndoType& insertUndo, c
return false;
}
void CClaimTrieCacheExpirationFork::initializeIncrement()
void CClaimTrieCacheExpirationFork::expirationForkActive(int nHeight, bool increment)
{
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
if (nNextHeight != Params().GetConsensus().nExtendedClaimExpirationForkHeight)
return;
forkForExpirationChange(true);
}
bool CClaimTrieCacheExpirationFork::finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
{
auto ret = CClaimTrieCacheBase::finalizeDecrement(takeoverHeightUndo);
if (ret && nNextHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight)
forkForExpirationChange(false);
return ret;
if (nHeight == Params().GetConsensus().nExtendedClaimExpirationForkHeight)
forkForExpirationChange(increment);
}
bool CClaimTrieCacheExpirationFork::forkForExpirationChange(bool increment)
@@ -172,12 +161,17 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary(insert
// run the one-time upgrade of all names that need to change
// it modifies the (cache) trie as it goes, so we need to grab everything to be modified first
for (auto it = base->cbegin(); it != base->cend(); ++it) {
const std::string normalized = normalizeClaimName(it.key(), true);
if (normalized == it.key())
boost::scoped_ptr<CDBIterator> pcursor(base->db->NewIterator());
for (pcursor->SeekToFirst(); pcursor->Valid(); pcursor->Next()) {
std::pair<uint8_t, std::string> key;
if (!pcursor->GetKey(key) || key.first != TRIE_NODE_BY_NAME)
continue;
const auto& name = key.second;
const std::string normalized = normalizeClaimName(name, true);
if (normalized == key.second)
continue;
auto& name = it.key();
auto supports = getSupportsForName(name);
for (auto support : supports) {
// if it's already going to expire just skip it
@@ -239,7 +233,7 @@ bool CClaimTrieCacheNormalizationFork::getInfoForName(const std::string& name, C
return CClaimTrieCacheExpirationFork::getInfoForName(normalizeClaimName(name), claim);
}
CClaimSupportToName CClaimTrieCacheNormalizationFork::getClaimsForName(const std::string& name) const
CClaimsForNameType CClaimTrieCacheNormalizationFork::getClaimsForName(const std::string& name) const
{
return CClaimTrieCacheExpirationFork::getClaimsForName(normalizeClaimName(name));
}
@@ -253,229 +247,3 @@ std::string CClaimTrieCacheNormalizationFork::adjustNameForValidHeight(const std
{
return normalizeClaimName(name, validHeight > Params().GetConsensus().nNormalizedNameForkHeight);
}
CClaimTrieCacheHashFork::CClaimTrieCacheHashFork(CClaimTrie* base) : CClaimTrieCacheNormalizationFork(base)
{
}
static const uint256 leafHash = uint256S("0000000000000000000000000000000000000000000000000000000000000002");
static const uint256 emptyHash = uint256S("0000000000000000000000000000000000000000000000000000000000000003");
std::vector<uint256> getClaimHashes(const CClaimTrieData& data)
{
std::vector<uint256> hashes;
for (auto& claim : data.claims)
hashes.push_back(getValueHash(claim.outPoint, data.nHeightOfLastTakeover));
return hashes;
}
template <typename T>
using iCbType = std::function<uint256(T&)>;
template <typename TIterator>
uint256 recursiveBinaryTreeHash(TIterator& it, const iCbType<TIterator>& process)
{
std::vector<uint256> childHashes;
for (auto& child : it.children())
childHashes.emplace_back(process(child));
std::vector<uint256> claimHashes;
if (!it->empty())
claimHashes = getClaimHashes(it.data());
else if (!it.hasChildren())
return {};
auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes);
return Hash(left.begin(), left.end(), right.begin(), right.end());
}
uint256 CClaimTrieCacheHashFork::recursiveComputeMerkleHash(CClaimTrie::iterator& it)
{
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::recursiveComputeMerkleHash(it);
using iterator = CClaimTrie::iterator;
iCbType<iterator> process = [&process](iterator& it) -> uint256 {
if (it->hash.IsNull())
it->hash = recursiveBinaryTreeHash(it, process);
assert(!it->hash.IsNull());
return it->hash;
};
return process(it);
}
bool CClaimTrieCacheHashFork::recursiveCheckConsistency(CClaimTrie::const_iterator& it, std::string& failed) const
{
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::recursiveCheckConsistency(it, failed);
struct CRecursiveBreak {};
using iterator = CClaimTrie::const_iterator;
iCbType<iterator> process = [&failed, &process](iterator& it) -> uint256 {
if (it->hash.IsNull() || it->hash != recursiveBinaryTreeHash(it, process)) {
failed = it.key();
throw CRecursiveBreak();
}
return it->hash;
};
try {
process(it);
} catch (const CRecursiveBreak&) {
return false;
}
return true;
}
std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
{
uint32_t count = 0;
int matchlevel = -1;
bool matchh = false;
uint256 inner[32], h;
const uint32_t one = 1;
std::vector<uint256> res;
const auto iterateInner = [&](int& level) {
for (; !(count & (one << level)); level++) {
const auto& ihash = inner[level];
if (matchh) {
res.push_back(ihash);
} else if (matchlevel == level) {
res.push_back(h);
matchh = true;
}
h = Hash(ihash.begin(), ihash.end(), h.begin(), h.end());
}
};
while (count < hashes.size()) {
h = hashes[count];
matchh = count == idx;
count++;
int level = 0;
iterateInner(level);
// Store the resulting hash at inner position level.
inner[level] = h;
if (matchh)
matchlevel = level;
}
int level = 0;
while (!(count & (one << level)))
level++;
h = inner[level];
matchh = matchlevel == level;
while (count != (one << level)) {
// If we reach this point, h is an inner value that is not the top.
if (matchh)
res.push_back(h);
h = Hash(h.begin(), h.end(), h.begin(), h.end());
// Increment count to the value it would have if two entries at this
count += (one << level);
level++;
iterateInner(level);
}
return res;
}
bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof)
{
return getProofForName(name, proof, nullptr);
}
bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, CClaimTrieProof& proof, const std::function<bool(const CClaimValue&)>& comp)
{
if (nNextHeight < Params().GetConsensus().nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::getProofForName(name, proof);
auto fillPairs = [&proof](const std::vector<uint256>& hashes, uint32_t idx) {
auto partials = ComputeMerklePath(hashes, idx);
for (int i = partials.size() - 1; i >= 0; --i)
proof.pairs.emplace_back((idx >> i) & 1, partials[i]);
};
// cache the parent nodes
cacheData(name, false);
getMerkleHash();
proof = CClaimTrieProof();
for (auto& it : static_cast<const CClaimTrie&>(nodesToAddOrUpdate).nodes(name)) {
std::vector<uint256> childHashes;
uint32_t nextCurrentIdx = 0;
for (auto& child : it.children()) {
if (name.find(child.key()) == 0)
nextCurrentIdx = uint32_t(childHashes.size());
childHashes.push_back(child->hash);
}
std::vector<uint256> claimHashes;
if (!it->empty())
claimHashes = getClaimHashes(it.data());
// I am on a node; I need a hash(children, claims)
// if I am the last node on the list, it will be hash(children, x)
// else it will be hash(x, claims)
if (it.key() == name) {
uint32_t nClaimIndex = 0;
auto& claims = it->claims;
auto itClaim = !comp ? claims.begin() : std::find_if(claims.begin(), claims.end(), comp);
if (itClaim != claims.end()) {
proof.hasValue = true;
proof.outPoint = itClaim->outPoint;
proof.nHeightOfLastTakeover = it->nHeightOfLastTakeover;
nClaimIndex = std::distance(claims.begin(), itClaim);
}
auto hash = childHashes.empty() ? leafHash : ComputeMerkleRoot(childHashes);
proof.pairs.emplace_back(true, hash);
if (!claimHashes.empty())
fillPairs(claimHashes, nClaimIndex);
} else {
auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(claimHashes);
proof.pairs.emplace_back(false, hash);
if (!childHashes.empty())
fillPairs(childHashes, nextCurrentIdx);
}
}
std::reverse(proof.pairs.begin(), proof.pairs.end());
return true;
}
void CClaimTrieCacheHashFork::copyAllBaseToCache()
{
for (auto it = base->cbegin(); it != base->cend(); ++it)
if (nodesAlreadyCached.insert(it.key()).second)
nodesToAddOrUpdate.insert(it.key(), it.data());
for (auto it = nodesToAddOrUpdate.begin(); it != nodesToAddOrUpdate.end(); ++it)
it->hash.SetNull();
}
void CClaimTrieCacheHashFork::initializeIncrement()
{
CClaimTrieCacheNormalizationFork::initializeIncrement();
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
if (nNextHeight != Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
return;
// if we are forking, we load the entire base trie into the cache trie
// we reset its hash computation so it can be recomputed completely
copyAllBaseToCache();
}
bool CClaimTrieCacheHashFork::finalizeDecrement(std::vector<std::pair<std::string, int>>& takeoverHeightUndo)
{
auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement(takeoverHeightUndo);
if (ret && nNextHeight == Params().GetConsensus().nAllClaimsInMerkleForkHeight - 1)
copyAllBaseToCache();
return ret;
}
bool CClaimTrieCacheHashFork::allowSupportMetadata() const
{
return nNextHeight >= Params().GetConsensus().nAllClaimsInMerkleForkHeight;
}
-4
View File
@@ -81,8 +81,6 @@ struct Params {
int nMinTakeoverWorkaroundHeight;
int nMaxTakeoverWorkaroundHeight;
int nWitnessForkHeight;
int64_t nPowTargetSpacing;
int64_t nPowTargetTimespan;
/** how long it took claims to expire before the hard fork */
@@ -96,8 +94,6 @@ struct Params {
nOriginalClaimExpirationTime :
nExtendedClaimExpirationTime;
}
/** blocks before the hard fork that adds all claims into the merkle hash */
int64_t nAllClaimsInMerkleForkHeight;
int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; }
uint256 nMinimumChainWork;
uint256 defaultAssumeValid;
+1 -2
View File
@@ -154,8 +154,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i
const Coin& coin = inputs.AccessCoin(tx.vin[i].prevout);
assert(!coin.IsSpent());
const CTxOut &prevout = coin.out;
const CScript& scriptPubKey = StripClaimScriptPrefix(prevout.scriptPubKey);
nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, scriptPubKey, &tx.vin[i].scriptWitness, flags);
nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags);
}
return nSigOps;
}
+15 -33
View File
@@ -15,7 +15,6 @@
#include <util.h>
#include <utilmoneystr.h>
#include <utilstrencodings.h>
#include <nameclaim.h>
UniValue ValueFromAmount(const CAmount& amount)
{
@@ -148,20 +147,12 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_address)
out.pushKV("hex", HexStr(script.begin(), script.end()));
std::vector<std::vector<unsigned char>> solns;
txnouttype type; int claimOp;
auto stripped = StripClaimScriptPrefix(script, claimOp);
Solver(stripped, type, solns);
if (claimOp >= 0) {
out.pushKV("isclaim", UniValue(claimOp == OP_CLAIM_NAME || claimOp == OP_UPDATE_CLAIM));
out.pushKV("issupport", UniValue(claimOp == OP_SUPPORT_CLAIM));
out.pushKV("subtype", GetTxnOutputType(type));
out.pushKV("type", GetTxnOutputType(TX_NONSTANDARD)); // trying to keep backwards compatibility
}
else
out.pushKV("type", GetTxnOutputType(type)); // trying to keep backwards compatibility
txnouttype type;
Solver(script, type, solns);
out.pushKV("type", GetTxnOutputType(type));
CTxDestination address;
if (include_address && ExtractDestination(stripped, address)) {
if (include_address && ExtractDestination(script, address)) {
out.pushKV("address", EncodeDestination(address));
}
}
@@ -177,28 +168,19 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey,
if (fIncludeHex)
out.pushKV("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end()));
int claimOp;
auto stripped = StripClaimScriptPrefix(scriptPubKey, claimOp);
auto extracted = ExtractDestinations(stripped, type, addresses, nRequired);
if (extracted)
out.pushKV("reqSigs", nRequired);
if (claimOp >= 0) {
out.pushKV("isclaim", UniValue(claimOp == OP_CLAIM_NAME || claimOp == OP_UPDATE_CLAIM));
out.pushKV("issupport", UniValue(claimOp == OP_SUPPORT_CLAIM));
out.pushKV("subtype", GetTxnOutputType(type));
out.pushKV("type", GetTxnOutputType(TX_NONSTANDARD));
}
else
if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) {
out.pushKV("type", GetTxnOutputType(type));
if (extracted) {
UniValue a(UniValue::VARR);
for (const CTxDestination &addr : addresses) {
a.push_back(EncodeDestination(addr));
}
out.pushKV("addresses", a);
return;
}
out.pushKV("reqSigs", nRequired);
out.pushKV("type", GetTxnOutputType(type));
UniValue a(UniValue::VARR);
for (const CTxDestination& addr : addresses) {
a.push_back(EncodeDestination(addr));
}
out.pushKV("addresses", a);
}
void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags)
+41 -14
View File
@@ -80,14 +80,14 @@ static void SetMaxOpenFiles(leveldb::Options *options) {
// implementation that does not use extra file descriptors (the fds are
// closed after being mmaped).
//
// Increasing the value beyond the nmap count is dangerous because LevelDB will
// fall back to a non-mmap implementation when the file count is too large (thus contending select()).
// Increasing the value beyond the default is dangerous because LevelDB will
// fall back to a non-mmap implementation when the file count is too large.
// On 32-bit Unix host we should decrease the value because the handles use
// up real fds, and we want to avoid fd exhaustion issues.
//
// See PR #12495 for further discussion.
int default_open_files = 400;
int default_open_files = options->max_open_files;
#ifndef WIN32
if (sizeof(void*) < 8) {
options->max_open_files = 64;
@@ -97,13 +97,47 @@ static void SetMaxOpenFiles(leveldb::Options *options) {
options->max_open_files, default_open_files);
}
class CappedLenCache: public leveldb::Cache {
leveldb::Cache* inner;
std::size_t maxKeyLen;
public:
CappedLenCache(std::size_t capacity, std::size_t maxKeyLen)
: inner(leveldb::NewLRUCache(capacity)), maxKeyLen(maxKeyLen) {}
~CappedLenCache() override { delete inner; }
Handle* Insert(const leveldb::Slice& key, void* value, size_t charge,
void (*deleter)(const leveldb::Slice& key, void* value)) override {
if (key.size() <= maxKeyLen)
return inner->Insert(key, value, charge, deleter);
deleter(key, value);
return nullptr;
}
Handle* Lookup(const leveldb::Slice& key) override { return inner->Lookup(key); }
void Release(Handle* handle) override { return inner->Release(handle); }
void* Value(Handle* handle) override { return inner->Value(handle); }
void Erase(const leveldb::Slice& key) override {return inner->Erase(key); }
uint64_t NewId() override { return inner->NewId(); }
};
static leveldb::Options GetOptions(size_t nCacheSize)
{
leveldb::Options options;
auto write_cache = std::min(nCacheSize / 4, size_t(32 * 1024 * 1024)); // cap write_cache
options.filter_policy=leveldb::NewBloomFilterPolicy2(16);
options.write_buffer_size=60 * 1024 * 1024;
options.total_leveldb_mem=2500ULL * 1024ULL * 1024ULL;
options.env=leveldb::Env::Default();
options.compression = leveldb::kNoCompression;
options.info_log = new CBitcoinLevelDBLogger();
return options;
auto write_cache = std::min(nCacheSize / 4, size_t(4 * 1024 * 1024)); // cap write_cache at 4MB (default)
options.block_cache = leveldb::NewLRUCache(nCacheSize - write_cache * 2);
// options.block_cache = new CappedLenCache(nCacheSize - write_cache * 2, 6);
options.write_buffer_size = write_cache; // up to two write buffers may be held in memory simultaneously
options.filter_policy = leveldb::NewBloomFilterPolicy(12);
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
options.compression = leveldb::kNoCompression;
options.info_log = new CBitcoinLevelDBLogger();
if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
@@ -112,11 +146,12 @@ static leveldb::Options GetOptions(size_t nCacheSize)
options.paranoid_checks = true;
}
SetMaxOpenFiles(&options);
options.max_open_files = 30000;
return options;
}
CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
: m_name(fs::basename(path)), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION)
: m_name(fs::basename(path))
{
penv = nullptr;
readoptions.verify_checksums = true;
@@ -181,16 +216,8 @@ CDBWrapper::~CDBWrapper()
options.env = nullptr;
}
bool CDBWrapper::Sync() {
CDBBatch batch(*this);
return WriteBatch(batch, true);
}
bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
{
if (!pdb)
return false;
const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
double mem_before = 0;
if (log_memory) {
+104 -102
View File
@@ -16,6 +16,9 @@
#include <leveldb/db.h>
#include <leveldb/write_batch.h>
static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
class dbwrapper_error : public std::runtime_error
{
public:
@@ -30,16 +33,87 @@ namespace dbwrapper_private {
/** Handle database error by throwing dbwrapper_error exception.
*/
void HandleError(const leveldb::Status& status);
void HandleError(const leveldb::Status& status);
/** Work around circular dependency, as well as for testing in dbwrapper_tests.
* Database obfuscation should be considered an implementation detail of the
* specific database.
*/
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
};
/** Batch of changes queued to be written to a CDBWrapper */
class CDBBatch
{
friend class CDBWrapper;
private:
const CDBWrapper &parent;
leveldb::WriteBatch batch;
CDataStream ssKey;
CDataStream ssValue;
size_t size_estimate;
public:
/**
* @param[in] _parent CDBWrapper that this batch is to be submitted to
*/
explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { };
void Clear()
{
batch.Clear();
size_estimate = 0;
}
template <typename K, typename V>
void Write(const K& key, const V& value)
{
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE);
ssValue << value;
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
leveldb::Slice slValue(ssValue.data(), ssValue.size());
batch.Put(slKey, slValue, nullptr);
// LevelDB serializes writes as:
// - byte: header
// - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
// - byte[]: key
// - varint: value length
// - byte[]: value
// The formula below assumes the key and value are both less than 16k.
size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
ssKey.clear();
ssValue.clear();
}
template <typename K>
void Erase(const K& key)
{
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
batch.Delete(slKey);
// LevelDB serializes erases as:
// - byte: header
// - varint: key length
// - byte[]: key
// The formula below assumes the key is less than 16kB.
size_estimate += 2 + (slKey.size() > 127) + slKey.size();
ssKey.clear();
}
size_t SizeEstimate() const { return size_estimate; }
};
class CDBIterator
{
private:
@@ -53,7 +127,7 @@ public:
* @param[in] _piter The original leveldb iterator.
*/
CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) :
parent(_parent), piter(_piter) { };
parent(_parent), piter(_piter) { };
~CDBIterator();
bool Valid() const;
@@ -62,6 +136,7 @@ public:
template<typename K> void Seek(const K& key) {
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
piter->Seek(slKey);
@@ -98,8 +173,6 @@ public:
};
class CDBBatch;
class CDBWrapper
{
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
@@ -140,8 +213,6 @@ private:
std::vector<unsigned char> CreateObfuscateKey() const;
public:
mutable CDataStream ssKey, ssValue;
/**
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
@@ -151,7 +222,7 @@ public:
* with a zero'd byte array.
*/
CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
virtual ~CDBWrapper();
~CDBWrapper();
CDBWrapper(const CDBWrapper&) = delete;
/* CDBWrapper& operator=(const CDBWrapper&) = delete; */
@@ -159,13 +230,13 @@ public:
template <typename K, typename V>
bool Read(const K& key, V& value) const
{
assert(ssKey.empty());
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
std::string strValue;
leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
ssKey.clear();
if (!status.ok()) {
if (status.IsNotFound())
return false;
@@ -183,17 +254,23 @@ public:
}
template <typename K, typename V>
bool Write(const K& key, const V& value, bool fSync = false);
bool Write(const K& key, const V& value, bool fSync = false)
{
CDBBatch batch(*this);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
template <typename K>
bool Exists(const K& key) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
std::string strValue;
leveldb::Status status = pdb->Get(readoptions, slKey, &strValue);
ssKey.clear();
if (!status.ok()) {
if (status.IsNotFound())
return false;
@@ -204,7 +281,12 @@ public:
}
template <typename K>
bool Erase(const K& key, bool fSync = false);
bool Erase(const K& key, bool fSync = false)
{
CDBBatch batch(*this);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
bool WriteBatch(CDBBatch& batch, bool fSync = false);
@@ -217,7 +299,11 @@ public:
return true;
}
bool Sync();
bool Sync()
{
CDBBatch batch(*this);
return WriteBatch(batch, true);
}
CDBIterator *NewIterator()
{
@@ -233,8 +319,8 @@ public:
size_t EstimateSize(const K& key_begin, const K& key_end) const
{
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
ssKey1.reserve(ssKey.capacity());
ssKey2.reserve(ssKey.capacity());
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey1 << key_begin;
ssKey2 << key_end;
leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
@@ -252,8 +338,8 @@ public:
void CompactRange(const K& key_begin, const K& key_end) const
{
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
ssKey1.reserve(ssKey.capacity());
ssKey2.reserve(ssKey.capacity());
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey1 << key_begin;
ssKey2 << key_end;
leveldb::Slice slKey1(ssKey1.data(), ssKey1.size());
@@ -263,88 +349,4 @@ public:
};
/** Batch of changes queued to be written to a CDBWrapper */
class CDBBatch
{
friend class CDBWrapper;
const CDBWrapper &parent;
leveldb::WriteBatch batch;
size_t size_estimate;
CDataStream ssKey, ssValue;
public:
/**
* @param[in] _parent CDBWrapper that this batch is to be submitted to
*/
explicit CDBBatch(const CDBWrapper &_parent) : parent(_parent), size_estimate(0),
ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) {
ssKey.reserve(parent.ssKey.capacity());
ssValue.reserve(parent.ssValue.capacity());
};
void Clear()
{
batch.Clear();
size_estimate = 0;
}
template <typename K, typename V>
void Write(const K& key, const V& value)
{
assert(ssKey.empty());
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
assert(ssValue.empty());
ssValue << value;
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
leveldb::Slice slValue(ssValue.data(), ssValue.size());
batch.Put(slKey, slValue);
// LevelDB serializes writes as:
// - byte: header
// - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
// - byte[]: key
// - varint: value length
// - byte[]: value
// The formula below assumes the key and value are both less than 16k.
size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
ssKey.clear();
ssValue.clear();
}
template <typename K>
void Erase(const K& key)
{
ssKey << key;
leveldb::Slice slKey(ssKey.data(), ssKey.size());
batch.Delete(slKey);
// LevelDB serializes erases as:
// - byte: header
// - varint: key length
// - byte[]: key
// The formula below assumes the key is less than 16kB.
size_estimate += 2 + (slKey.size() > 127) + slKey.size();
ssKey.clear();
}
size_t SizeEstimate() const { return size_estimate; }
};
template<typename K>
bool CDBWrapper::Erase(const K &key, bool fSync) {
CDBBatch batch(*this);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
template<typename K, typename V>
bool CDBWrapper::Write(const K &key, const V &value, bool fSync) {
CDBBatch batch(*this);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
#endif // BITCOIN_DBWRAPPER_H
-2
View File
@@ -28,8 +28,6 @@ protected:
DB(const fs::path& path, size_t n_cache_size,
bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
~DB() override {}
/// Read block locator of the chain that the txindex is in sync with.
bool ReadBestBlock(CBlockLocator& locator) const;
-1
View File
@@ -29,7 +29,6 @@ class TxIndex::DB : public BaseIndex::DB
{
public:
explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
~DB() override {}
/// Read the disk location of the transaction data with the given hash. Returns false if the
/// transaction hash is not indexed.
+7 -9
View File
@@ -22,7 +22,6 @@
#include <httprpc.h>
#include <index/txindex.h>
#include <key.h>
#include <lbry.h>
#include <validation.h>
#include <miner.h>
#include <netbase.h>
@@ -371,7 +370,6 @@ void SetupServerArgs()
gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS);
gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS);
gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-claimtriecache=<n>", strprintf("Set claim trie cache size in megabytes (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS);
gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS);
@@ -399,7 +397,6 @@ void SetupServerArgs()
hidden_args.emplace_back("-sysperms");
#endif
gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-memfile=<GiB>", "Use a memory mapped file for the claimtrie allocations (default: use RAM instead)", false, OptionsCategory::OPTIONS);
gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
@@ -486,6 +483,7 @@ void SetupServerArgs()
CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST);
gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), true, OptionsCategory::DEBUG_TEST);
gArgs.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", false, OptionsCategory::DEBUG_TEST);
gArgs.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", false, OptionsCategory::DEBUG_TEST);
gArgs.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", false, OptionsCategory::DEBUG_TEST);
SetupChainParamsBaseOptions();
@@ -1235,6 +1233,11 @@ bool AppInitMain()
CreatePidFile(GetPidFile(), getpid());
#endif
if (g_logger->m_print_to_file) {
if (gArgs.GetBoolArg("-shrinkdebugfile", g_logger->DefaultShrinkDebugFile())) {
// Do this first since it both loads a bunch of debug.log into memory,
// and because this needs to happen before any other debug.log printing
g_logger->ShrinkDebugFile();
}
if (!g_logger->OpenDebugLog()) {
return InitError(strprintf("Could not open debug log file %s",
g_logger->m_file_path.string()));
@@ -1437,8 +1440,6 @@ bool AppInitMain()
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
g_memfileSize = gArgs.GetArg("-memfile", 0u);
bool fLoaded = false;
while (!fLoaded && !ShutdownRequested()) {
bool fReset = fReindex;
@@ -1460,10 +1461,7 @@ bool AppInitMain()
pblocktree.reset();
pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset));
delete pclaimTrie;
int64_t trieCacheMB = gArgs.GetArg("-claimtriecache", nDefaultDbCache);
trieCacheMB = std::min(trieCacheMB, nMaxDbCache);
trieCacheMB = std::max(trieCacheMB, nMinDbCache);
pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState, 32, trieCacheMB);
pclaimTrie = new CClaimTrie(false, fReindex || fReindexChainState);
if (fReset) {
pblocktree->WriteReindexing(true);
-2
View File
@@ -3,8 +3,6 @@
#include <cstdio>
uint32_t g_memfileSize = 0;
unsigned int CalculateLbryNextWorkRequired(const CBlockIndex* pindexLast, int64_t nFirstBlockTime, const Consensus::Params& params)
{
if (params.fPowNoRetargeting)
-1
View File
@@ -4,7 +4,6 @@
#include <chain.h>
#include <chainparams.h>
extern uint32_t g_memfileSize;
unsigned int CalculateLbryNextWorkRequired(const CBlockIndex* pindexLast, int64_t nLastRetargetTime, const Consensus::Params& params);
#endif
-13
View File
@@ -1,13 +0,0 @@
build_config.mk
*.a
*.o
*.dylib*
*.so
*.so.*
*_test
db_bench
leveldbutil
Release
Debug
Benchmark
vs2010.*
-4
View File
@@ -6,7 +6,3 @@ Google Inc.
# Initial version authors:
Jeffrey Dean <jeff@google.com>
Sanjay Ghemawat <sanjay@google.com>
# Partial list of contributors:
Kevin Regan <kevin.d.regan@gmail.com>
Johan Bilien <jobi@litl.com>
+72
View File
@@ -0,0 +1,72 @@
github.com tag 2.0.34 - February 15, 2017
-----------------------------------------
mv-hot-backup2: - correct MakeTieredDbname() within db/filename.cc
for case where dbname input is blank and fast/slow
already populated in options. Corrects issue
with hot backup in non-tiered storage situations
github.com tag 2.0.33 - November 21, 2016
-----------------------------------------
mv-bucket-expiry: - partial branch to enable X-Riak-Meta-Expiry-Base-Seconds
property within enterprise edition
--- no 2.0.32 tag on leveldb ---
github.com tag 2.0.31 - November 1, 2016
----------------------------------------
- version shipped with Riak 2.2
mv-no-md-expiry: - Riak specific
- never convert a key prefix of sext:encoded "{md" to expiry
- update sst_scan for dumping Riak formated keys
mv-tuning8: - rework penalty rules in version_set.cc UpdatePenalty()
- add unit test framework for UpdatePenalty()
github.com tag 2.0.30 - October 11, 2016
----------------------------------------
mv-delayed-bloom: - when opening an .sst table file, only load
bloom filter on second Get() operation. Saves time.
- correct VersionSet::Finalize() logic for level 1 when
when level 2 is above desired size
- move hot backup to Riak ee build
github.com tag 2.0.29 - September 13, 2016
------------------------------------------
mv-expiry-manifest: only switch to expiry enabled manifest format
if expiry function enabled. Eases downgrade
during early Riak releases containing expiry
github.com tag 2.0.28 - September 6, 2016
-----------------------------------------
mv-hot-backup: add externally triggered hot backup feature
github.com tag 2.0.27 - August 22, 2016
---------------------------------------
mv-mem-fences: fix iterator double delete bug in eleveldb and
build better memory fenced operations for referenced count objects.
github.com tag 2.0.26 - August 21, 2016
---------------------------------------
mv-expiry-iter-bug: DBImpl::NewIterator() was not setting the new expiry parameter.
github.com tag 2.0.25 - August 10, 2016
---------------------------------------
Make LZ4 the default compression instead of Snappy.
github.com tag 2.0.24 - August 2, 2016
--------------------------------------
mv-expiry: open source expiry. Supports one expiry policy for all databases.
github.com tag 2.0.23 - July 20, 2016
-------------------------------------
mv-no-semaphore: remove semaphore controlled thread in hot_threads.cc. Instead use
use mutex of thread 0 (only one thread's mutex) to address know race condition.
github.com tag 2.0.22 - June 22, 2016
-------------------------------------
no change: iterator fix in eleveldb
github.com tag 2.0.21 - June 16, 2016
-------------------------------------
branch mv-iterator-hot-threads: correct condition where eleveldb MoveTask
could hang an iterator. (https://github.com/basho/leveldb/wiki/mv-iterator-hot-threads)
-36
View File
@@ -1,36 +0,0 @@
# Contributing
We'd love to accept your code patches! However, before we can take them, we
have to jump a couple of legal hurdles.
## Contributor License Agreements
Please fill out either the individual or corporate Contributor License
Agreement as appropriate.
* If you are an individual writing original source code and you're sure you
own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual).
* If you work for a company that wants to allow you to contribute your work,
then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate).
Follow either of the two links above to access the appropriate CLA and
instructions for how to sign and return it.
## Submitting a Patch
1. Sign the contributors license agreement above.
2. Decide which code you want to submit. A submission should be a set of changes
that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues).
Please don't mix more than one logical change per submission, because it makes
the history hard to follow. If you want to make a change
(e.g. add a sample or feature) that doesn't have a corresponding issue in the
issue tracker, please create one.
3. **Submitting**: When you are ready to submit, send us a Pull Request. Be
sure to include the issue number you fixed and the name you used to sign
the CLA.
## Writing Code ##
If your contribution contains code, please make sure that it follows
[the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml).
Otherwise we will have to ask you to make changes, and that's no fun for anyone.
+158 -362
View File
@@ -2,423 +2,219 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. See the AUTHORS file for names of contributors.
# Inherit some settings from environment variables, if available
INSTALL_PATH ?= $(CURDIR)
#-----------------------------------------------
# Uncomment exactly one of the lines labelled (A), (B), and (C) below
# to switch between compilation modes.
# NOTE: targets "debug" and "prof" provide same functionality
# NOTE 2: -DNDEBUG disables assert() statements within C code,
# i.e. no assert()s in production code
# (A) Production use (optimized mode)
OPT ?= -O2 -DNDEBUG
# (B) Debug mode, w/ full line-level debugging symbols
# OPT ?= -g2
# (C) Profiling mode: opt, but w/debugging symbols
# OPT ?= -O2 -g2 -DNDEBUG
OPT ?= -O2 -g -DNDEBUG # (A) Production use (optimized mode)
# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols
# OPT ?= -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols
#-----------------------------------------------
# detect what platform we're building on
$(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \
./build_detect_platform build_config.mk ./)
ifeq ($(wildcard build_config.mk),)
$(shell ./build_detect_platform build_config.mk)
endif
# this file is generated by the previous line to set build flags and sources
include build_config.mk
TESTS = \
db/autocompact_test \
db/c_test \
db/corruption_test \
db/db_test \
db/dbformat_test \
db/fault_injection_test \
db/filename_test \
db/log_test \
db/recovery_test \
db/skiplist_test \
db/version_edit_test \
db/version_set_test \
db/write_batch_test \
helpers/memenv/memenv_test \
issues/issue178_test \
issues/issue200_test \
table/filter_block_test \
table/table_test \
util/arena_test \
util/bloom_test \
util/cache_test \
util/coding_test \
util/crc32c_test \
util/env_posix_test \
util/env_test \
util/hash_test
UTILS = \
db/db_bench \
db/leveldbutil
# Put the object files in a subdirectory, but the application at the top of the object dir.
PROGNAMES := $(notdir $(TESTS) $(UTILS))
# On Linux may need libkyotocabinet-dev for dependency.
BENCHMARKS = \
doc/bench/db_bench_sqlite3 \
doc/bench/db_bench_tree_db
CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)
LDFLAGS += $(PLATFORM_LDFLAGS)
LIBS += $(PLATFORM_LIBS)
SIMULATOR_OUTDIR=out-ios-x86
DEVICE_OUTDIR=out-ios-arm
LIBOBJECTS := $(SOURCES:.cc=.o)
LIBOBJECTS += util/lz4.o
MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
DEPEND := $(SOURCES:.cc=.d)
ifeq ($(PLATFORM), IOS)
# Note: iOS should probably be using libtool, not ar.
AR=xcrun ar
SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path)
DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path)
DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64
SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64
STATIC_OUTDIR=out-ios-universal
TESTUTIL = ./util/testutil.o
TESTHARNESS = ./util/testharness.o $(TESTUTIL)
TESTS := $(sort $(notdir $(basename $(TEST_SOURCES))))
TOOLS = \
leveldb_repair \
perf_dump \
sst_rewrite \
sst_scan
PROGRAMS = db_bench $(TESTS) $(TOOLS)
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
LIBRARY = libleveldb.a
MEMENVLIBRARY = libmemenv.a
#
# static link leveldb to tools to simplify platform usage (if Linux)
#
ifeq ($(PLATFORM),OS_LINUX)
LEVEL_LDFLAGS := -L . -Wl,-non_shared -lleveldb -Wl,-call_shared
else
STATIC_OUTDIR=out-static
SHARED_OUTDIR=out-shared
STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES))
SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench)
LEVEL_LDFLAGS := -L . -lleveldb
endif
STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o))
STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o))
DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o))
SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o))
SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o
TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL)
STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS)))
STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS)))
STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS)
DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS)
SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS)
default: all
# Should we build shared libraries?
ifneq ($(PLATFORM_SHARED_EXT),)
# Many leveldb test apps use non-exported API's. Only build a subset for testing.
SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS)
ifneq ($(PLATFORM_SHARED_VERSIONED),true)
SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED_LIB2 = $(SHARED_LIB1)
SHARED_LIB3 = $(SHARED_LIB1)
SHARED_LIBS = $(SHARED_LIB1)
SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1)
SHARED3 = $(SHARED1)
SHARED = $(SHARED1)
else
# Update db.h if you change these.
SHARED_VERSION_MAJOR = 1
SHARED_VERSION_MINOR = 20
SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR)
SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR)
SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3)
$(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3)
ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1)
$(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3)
ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2)
SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
SHARED_MAJOR = 1
SHARED_MINOR = 9
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
$(SHARED1): $(SHARED3)
ln -fs $(SHARED3) $(SHARED1)
$(SHARED2): $(SHARED3)
ln -fs $(SHARED3) $(SHARED2)
endif
$(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS)
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS)
$(SHARED3): $(LIBOBJECTS)
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIBOBJECTS) -o $(SHARED3) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2)
endif # PLATFORM_SHARED_EXT
all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS)
all: $(SHARED) $(LIBRARY)
check: $(STATIC_PROGRAMS)
for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done
test check: all $(PROGRAMS) $(TESTS)
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
clean:
-rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal
-rm -f build_config.mk
-rm -rf ios-x86 ios-arm
tools: all $(TOOLS)
$(STATIC_OUTDIR):
mkdir $@
$(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR)
mkdir $@
$(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR)
mkdir -p $@
$(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR)
mkdir $@
$(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR)
mkdir $@
$(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR)
mkdir $@
.PHONY: STATIC_OBJDIRS
STATIC_OBJDIRS: \
$(STATIC_OUTDIR)/db \
$(STATIC_OUTDIR)/port \
$(STATIC_OUTDIR)/table \
$(STATIC_OUTDIR)/util \
$(STATIC_OUTDIR)/helpers/memenv
$(SHARED_OUTDIR):
mkdir $@
$(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR)
mkdir $@
$(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR)
mkdir -p $@
$(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR)
mkdir $@
$(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR)
mkdir $@
$(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR)
mkdir $@
.PHONY: SHARED_OBJDIRS
SHARED_OBJDIRS: \
$(SHARED_OUTDIR)/db \
$(SHARED_OUTDIR)/port \
$(SHARED_OUTDIR)/table \
$(SHARED_OUTDIR)/util \
$(SHARED_OUTDIR)/helpers/memenv
$(DEVICE_OUTDIR):
mkdir $@
$(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR)
mkdir $@
$(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR)
mkdir -p $@
$(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR)
mkdir $@
$(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR)
mkdir $@
$(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR)
mkdir $@
.PHONY: DEVICE_OBJDIRS
DEVICE_OBJDIRS: \
$(DEVICE_OUTDIR)/db \
$(DEVICE_OUTDIR)/port \
$(DEVICE_OUTDIR)/table \
$(DEVICE_OUTDIR)/util \
$(DEVICE_OUTDIR)/helpers/memenv
$(SIMULATOR_OUTDIR):
mkdir $@
$(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR)
mkdir $@
$(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR)
mkdir -p $@
$(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR)
mkdir $@
$(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR)
mkdir $@
$(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR)
mkdir $@
.PHONY: SIMULATOR_OBJDIRS
SIMULATOR_OBJDIRS: \
$(SIMULATOR_OUTDIR)/db \
$(SIMULATOR_OUTDIR)/port \
$(SIMULATOR_OUTDIR)/table \
$(SIMULATOR_OUTDIR)/util \
$(SIMULATOR_OUTDIR)/helpers/memenv
$(STATIC_ALLOBJS): | STATIC_OBJDIRS
$(DEVICE_ALLOBJS): | DEVICE_OBJDIRS
$(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS
$(SHARED_ALLOBJS): | SHARED_OBJDIRS
ifeq ($(PLATFORM), IOS)
$(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS)
rm -f $@
$(AR) -rs $@ $(DEVICE_LIBOBJECTS)
$(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS)
rm -f $@
$(AR) -rs $@ $(SIMULATOR_LIBOBJECTS)
$(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS)
rm -f $@
$(AR) -rs $@ $(DEVICE_MEMENVOBJECTS)
$(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS)
rm -f $@
$(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS)
# For iOS, create universal object libraries to be used on both the simulator and
# a device.
$(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a
lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@
$(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a
lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@
else
$(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS)
rm -f $@
$(AR) -rs $@ $(STATIC_LIBOBJECTS)
$(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS)
rm -f $@
$(AR) -rs $@ $(STATIC_MEMENVOBJECTS)
#
# command line targets: debug and prof
# just like
ifneq ($(filter debug,$(MAKECMDGOALS)),)
OPT := -g2 # (B) Debug mode, w/ full line-level debugging symbols
debug: all
endif
$(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS)
ifneq ($(filter prof,$(MAKECMDGOALS)),)
OPT := -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols
prof: all
endif
clean:
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o */*.d */*/*.d ios-x86/*/*.o ios-arm/*/*.o build_config.mk include/leveldb/ldb_config.h
-rm -rf ios-x86/* ios-arm/* *.dSYM
$(LIBRARY): $(LIBOBJECTS)
rm -f $@
$(AR) -rs $@ $(SHARED_MEMENVOBJECTS)
$(AR) -rs $@ $(LIBOBJECTS)
$(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
#
# all tools, programs, and tests depend upon the static library
$(TESTS) $(PROGRAMS) $(TOOLS) : $(LIBRARY)
$(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
#
# all tests depend upon the test harness
$(TESTS) : $(TESTHARNESS)
$(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
#
# tools, programs, and tests will compile to the root directory
# but their .cc source file will be in one of the following subdirectories
vpath %.cc db:table:util:leveldb_ee:leveldb_os
$(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS)
# special case for c_test
vpath %.c db
$(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
db_bench: db/db_bench.o $(LIBRARY) $(TESTUTIL)
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< $(TESTUTIL) -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
$(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBRARY) $(TESTUTIL)
$(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBRARY) $(TESTUTIL)
$(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
#
# build line taken from lz4 makefile
#
util/lz4.o: util/lz4.c util/lz4.h
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -O3 -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"r130\" -c util/lz4.c -o util/lz4.o
$(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
#
# memory env
#
$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
rm -f $@
$(AR) -rs $@ $(MEMENVOBJECTS)
$(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LDFLAGS)
$(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
#
# IOS build
#
ifeq ($(PLATFORM), IOS)
# For iOS, create universal object files to be used on both the simulator and
# a device.
PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms
SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer
DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer
IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString)
$(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
.cc.o:
mkdir -p ios-x86/$(dir $@)
$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
mkdir -p ios-arm/$(dir $@)
$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
lipo ios-x86/$@ ios-arm/$@ -create -output $@
$(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
.c.o:
mkdir -p ios-x86/$(dir $@)
$(SIMULATORROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
mkdir -p ios-arm/$(dir $@)
$(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
lipo ios-x86/$@ ios-arm/$@ -create -output $@
$(STATIC_OUTDIR)/env_posix_test:util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS)
$(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS)
$(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL)
$(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS)
.PHONY: run-shared
run-shared: $(SHARED_OUTDIR)/db_bench
LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench
$(SIMULATOR_OUTDIR)/%.o: %.cc
xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
$(DEVICE_OUTDIR)/%.o: %.cc
xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
$(SIMULATOR_OUTDIR)/%.o: %.c
xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
$(DEVICE_OUTDIR)/%.o: %.c
xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
$(STATIC_OUTDIR)/%.o: %.cc
$(CXX) $(CXXFLAGS) -c $< -o $@
$(STATIC_OUTDIR)/%.o: %.c
$(CC) $(CFLAGS) -c $< -o $@
$(SHARED_OUTDIR)/%.o: %.cc
else
#
# build for everything NOT IOS
#
.cc.o:
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
$(SHARED_OUTDIR)/%.o: %.c
.c.o:
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
$(STATIC_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc
$(CXX) $(CXXFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@
## @echo -- Creating dependency file for $<
%.d: %.cc
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM -E -MT $(basename $@).d -MT $(basename $@).o -MF $@ $<
@echo $(basename $@).o: $(basename $@).d >>$@
$(SHARED_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@
# generic build for command line tests
%: %.cc
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< $(TESTHARNESS) -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
%: db/%.c
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< $(TESTHARNESS) -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
# for tools, omits test harness
%: tools/%.cc
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $< -o $@ $(LEVEL_LDFLAGS) $(LDFLAGS)
endif
#
# load dependency files
#
ifeq ($(filter tar clean allclean distclean,$(MAKECMDGOALS)),)
-include $(DEPEND)
endif
+83
View File
@@ -0,0 +1,83 @@
leveldb: A key-value store
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
The original Google README is now README.GOOGLE.
** Introduction
This repository contains the Google source code as modified to benefit
the Riak environment. The typical Riak environment has two attributes
that necessitate leveldb adjustments, both in options and code:
- production servers: Riak often runs in heavy Internet environments:
servers with many CPU cores, lots of memory, and 24x7 disk activity.
Basho's leveldb takes advantage of the environment by adding
hardware CRC calculation, increasing Bloom filter accuracy, and
defaulting to integrity checking enabled.
- multiple databases open: Riak opens 8 to 128 databases
simultaneously. Google's leveldb supports this, but its background
compaction thread can fall behind. leveldb will "stall" new user
writes whenever the compaction thread gets too far behind. Basho's
leveldb modification include multiple thread blocks that each
contain prioritized threads for specific compaction activities.
Details for Basho's customizations exist in the leveldb wiki:
http://github.com/basho/leveldb/wiki
** Branch pattern
This repository follows the Basho standard for branch management
as of November 28, 2013. The standard is found here:
https://github.com/basho/riak/wiki/Basho-repository-management
In summary, the "develop" branch contains the most recently reviewed
engineering work. The "master" branch contains the most recently
released work, i.e. distributed as part of a Riak release.
** Basic options needed
Those wishing to truly savor the benefits of Basho's modifications
need to initialize a new leveldb::Options structure similar to the
following before each call to leveldb::DB::Open:
leveldb::Options * options;
options=new Leveldb::Options;
options.filter_policy=leveldb::NewBloomFilterPolicy2(16);
options.write_buffer_size=62914560; // 60Mbytes
options.total_leveldb_mem=2684354560; // 2.5Gbytes (details below)
options.env=leveldb::Env::Default();
** Memory plan
Basho's leveldb dramatically departed from Google's original internal
memory allotment plan with Riak 2.0. Basho's leveldb uses a methodology
called flexcache. The technical details are here:
https://github.com/basho/leveldb/wiki/mv-flexcache
The key points are:
- options.total_leveldb_mem is an allocation for the entire process,
not a single database
- giving different values to options.total_leveldb_mem on subsequent Open
calls causes memory to rearrange to current value across all databases
- recommended minimum for Basho's leveldb is 340Mbytes per database.
- performance improves rapidly from 340Mbytes to 2.5Gbytes per database (3.0Gbytes
if using Riak's active anti-entropy). Even more is nice, but not as helpful.
- never assign more than 75% of available RAM to total_leveldb_mem. There is
too much unaccounted memory overhead (worse if you use tcmalloc library).
- options.max_open_files and options.block_cache should not be used.
+51
View File
@@ -0,0 +1,51 @@
leveldb: A key-value store
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
The code under this directory implements a system for maintaining a
persistent key/value store.
See doc/index.html for more explanation.
See doc/impl.html for a brief overview of the implementation.
The public interface is in include/*.h. Callers should not include or
rely on the details of any other header files in this package. Those
internal APIs may be changed without warning.
Guide to header files:
include/db.h
Main interface to the DB: Start here
include/options.h
Control over the behavior of an entire database, and also
control over the behavior of individual reads and writes.
include/comparator.h
Abstraction for user-specified comparison function. If you want
just bytewise comparison of keys, you can use the default comparator,
but clients can write their own comparator implementations if they
want custom ordering (e.g. to handle different character
encodings, etc.)
include/iterator.h
Interface for iterating over data. You can get an iterator
from a DB object.
include/write_batch.h
Interface for atomically applying multiple updates to a database.
include/slice.h
A simple module for maintaining a pointer and a length into some
other byte array.
include/status.h
Status is returned from many of the public interfaces and is used
to report success and various kinds of errors.
include/env.h
Abstraction of the OS environment. A posix implementation of
this interface is in util/env_posix.cc
include/table.h
include/table_builder.h
Lower-level modules that most clients probably won't use directly
-174
View File
@@ -1,174 +0,0 @@
**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
[![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb)
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
# Features
* Keys and values are arbitrary byte arrays.
* Data is stored sorted by key.
* Callers can provide a custom comparison function to override the sort order.
* The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`.
* Multiple changes can be made in one atomic batch.
* Users can create a transient snapshot to get a consistent view of data.
* Forward and backward iteration is supported over the data.
* Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/).
* External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
# Documentation
[LevelDB library documentation](https://github.com/google/leveldb/blob/master/doc/index.md) is online and bundled with the source code.
# Limitations
* This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes.
* Only a single process (possibly multi-threaded) can access a particular database at a time.
* There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library.
# Contributing to the leveldb Project
The leveldb project welcomes contributions. leveldb's primary goal is to be
a reliable and fast key/value store. Changes that are in line with the
features/limitations outlined above, and meet the requirements below,
will be considered.
Contribution requirements:
1. **POSIX only**. We _generally_ will only accept changes that are both
compiled, and tested on a POSIX platform - usually Linux. Very small
changes will sometimes be accepted, but consider that more of an
exception than the rule.
2. **Stable API**. We strive very hard to maintain a stable API. Changes that
require changes for projects using leveldb _might_ be rejected without
sufficient benefit to the project.
3. **Tests**: All changes must be accompanied by a new (or changed) test, or
a sufficient explanation as to why a new (or changed) test is not required.
## Submitting a Pull Request
Before any pull request will be accepted the author must first sign a
Contributor License Agreement (CLA) at https://cla.developers.google.com/.
In order to keep the commit timeline linear
[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits)
your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase)
on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed
with the internal repository at Google. More information at GitHub's
[About Git rebase](https://help.github.com/articles/about-git-rebase/) page.
# Performance
Here is a performance report (with explanations) from the run of the
included db_bench program. The results are somewhat noisy, but should
be enough to get a ballpark performance estimate.
## Setup
We use a database with a million entries. Each entry has a 16 byte
key, and a 100 byte value. Values used by the benchmark compress to
about half their original size.
LevelDB: version 1.1
Date: Sun May 1 12:11:26 2011
CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz
CPUCache: 4096 KB
Keys: 16 bytes each
Values: 100 bytes each (50 bytes after compression)
Entries: 1000000
Raw Size: 110.6 MB (estimated)
File Size: 62.9 MB (estimated)
## Write performance
The "fill" benchmarks create a brand new database, in either
sequential, or random order. The "fillsync" benchmark flushes data
from the operating system to the disk after every operation; the other
write operations leave the data sitting in the operating system buffer
cache for a while. The "overwrite" benchmark does random writes that
update existing keys in the database.
fillseq : 1.765 micros/op; 62.7 MB/s
fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops)
fillrandom : 2.460 micros/op; 45.0 MB/s
overwrite : 2.380 micros/op; 46.5 MB/s
Each "op" above corresponds to a write of a single key/value pair.
I.e., a random write benchmark goes at approximately 400,000 writes per second.
Each "fillsync" operation costs much less (0.3 millisecond)
than a disk seek (typically 10 milliseconds). We suspect that this is
because the hard disk itself is buffering the update in its memory and
responding before the data has been written to the platter. This may
or may not be safe based on whether or not the hard disk has enough
power to save its memory in the event of a power failure.
## Read performance
We list the performance of reading sequentially in both the forward
and reverse direction, and also the performance of a random lookup.
Note that the database created by the benchmark is quite small.
Therefore the report characterizes the performance of leveldb when the
working set fits in memory. The cost of reading a piece of data that
is not present in the operating system buffer cache will be dominated
by the one or two disk seeks needed to fetch the data from disk.
Write performance will be mostly unaffected by whether or not the
working set fits in memory.
readrandom : 16.677 micros/op; (approximately 60,000 reads per second)
readseq : 0.476 micros/op; 232.3 MB/s
readreverse : 0.724 micros/op; 152.9 MB/s
LevelDB compacts its underlying storage data in the background to
improve read performance. The results listed above were done
immediately after a lot of random writes. The results after
compactions (which are usually triggered automatically) are better.
readrandom : 11.602 micros/op; (approximately 85,000 reads per second)
readseq : 0.423 micros/op; 261.8 MB/s
readreverse : 0.663 micros/op; 166.9 MB/s
Some of the high cost of reads comes from repeated decompression of blocks
read from disk. If we supply enough cache to the leveldb so it can hold the
uncompressed blocks in memory, the read performance improves again:
readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction)
readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction)
## Repository contents
See [doc/index.md](doc/index.md) for more explanation. See
[doc/impl.md](doc/impl.md) for a brief overview of the implementation.
The public interface is in include/*.h. Callers should not include or
rely on the details of any other header files in this package. Those
internal APIs may be changed without warning.
Guide to header files:
* **include/db.h**: Main interface to the DB: Start here
* **include/options.h**: Control over the behavior of an entire database,
and also control over the behavior of individual reads and writes.
* **include/comparator.h**: Abstraction for user-specified comparison function.
If you want just bytewise comparison of keys, you can use the default
comparator, but clients can write their own comparator implementations if they
want custom ordering (e.g. to handle different character encodings, etc.)
* **include/iterator.h**: Interface for iterating over data. You can get
an iterator from a DB object.
* **include/write_batch.h**: Interface for atomically applying multiple
updates to a database.
* **include/slice.h**: A simple module for maintaining a pointer and a
length into some other byte array.
* **include/status.h**: Status is returned from many of the public interfaces
and is used to report success and various kinds of errors.
* **include/env.h**:
Abstraction of the OS environment. A posix implementation of this interface is
in util/env_posix.cc
* **include/table.h, include/table_builder.h**: Lower-level modules that most
clients probably won't use directly
-1
View File
@@ -7,7 +7,6 @@ db
within [start_key..end_key]? For Chrome, deletion of obsolete
object stores, etc. can be done in the background anyway, so
probably not that important.
- There have been requests for MultiGet.
After a range is completely deleted, what gets rid of the
corresponding files if we do no future changes to that range. Make
-39
View File
@@ -1,39 +0,0 @@
# Building LevelDB On Windows
## Prereqs
Install the [Windows Software Development Kit version 7.1](http://www.microsoft.com/downloads/dlx/en-us/listdetailsview.aspx?FamilyID=6b6c21d2-2006-4afa-9702-529fa782d63b).
Download and extract the [Snappy source distribution](http://snappy.googlecode.com/files/snappy-1.0.5.tar.gz)
1. Open the "Windows SDK 7.1 Command Prompt" :
Start Menu -> "Microsoft Windows SDK v7.1" > "Windows SDK 7.1 Command Prompt"
2. Change the directory to the leveldb project
## Building the Static lib
* 32 bit Version
setenv /x86
msbuild.exe /p:Configuration=Release /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5
* 64 bit Version
setenv /x64
msbuild.exe /p:Configuration=Release /p:Platform=x64 /p:Snappy=..\snappy-1.0.5
## Building and Running the Benchmark app
* 32 bit Version
setenv /x86
msbuild.exe /p:Configuration=Benchmark /p:Platform=Win32 /p:Snappy=..\snappy-1.0.5
Benchmark\leveldb.exe
* 64 bit Version
setenv /x64
msbuild.exe /p:Configuration=Benchmark /p:Platform=x64 /p:Snappy=..\snappy-1.0.5
x64\Benchmark\leveldb.exe
+98 -120
View File
@@ -7,11 +7,8 @@
# CC C Compiler path
# CXX C++ Compiler path
# PLATFORM_LDFLAGS Linker flags
# PLATFORM_LIBS Libraries flags
# PLATFORM_SHARED_EXT Extension for shared libraries
# PLATFORM_SHARED_LDFLAGS Flags for building shared library
# This flag is embedded just before the name
# of the shared library without intervening spaces
# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
# PLATFORM_CCFLAGS C compiler flags
# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
@@ -20,15 +17,14 @@
#
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
#
# -DLEVELDB_ATOMIC_PRESENT if <atomic> is present
# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present
# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
# -DSNAPPY if the Snappy library is present
#
OUTPUT=$1
PREFIX=$2
if test -z "$OUTPUT" || test -z "$PREFIX"; then
echo "usage: $0 <output-filename> <directory_prefix>" >&2
if test -z "$OUTPUT"; then
echo "usage: $0 <output-filename>" >&2
exit 1
fi
@@ -44,10 +40,6 @@ if test -z "$CXX"; then
CXX=g++
fi
if test -z "$TMPDIR"; then
TMPDIR=/tmp
fi
# Detect OS
if test -z "$TARGET_OS"; then
TARGET_OS=`uname -s`
@@ -58,119 +50,77 @@ CROSS_COMPILE=
PLATFORM_CCFLAGS=
PLATFORM_CXXFLAGS=
PLATFORM_LDFLAGS=
PLATFORM_LIBS=
PLATFORM_SHARED_EXT="so"
PLATFORM_SHARED_EXT=
PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
PLATFORM_SHARED_CFLAGS="-fPIC"
PLATFORM_SHARED_VERSIONED=true
PLATFORM_SSEFLAGS=
MEMCMP_FLAG=
if [ "$CXX" = "g++" ]; then
# Use libc's memcmp instead of GCC's memcmp. This results in ~40%
# performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
MEMCMP_FLAG="-fno-builtin-memcmp"
if test -n "$LEVELDB_VSN"; then
VERSION_FLAGS="$VERSION_FLAGS -DLEVELDB_VSN=\"$LEVELDB_VSN\""
fi
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
case "$TARGET_OS" in
CYGWIN_*)
PLATFORM=OS_LINUX
COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN"
PLATFORM_LDFLAGS="-lpthread"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
Darwin)
PLATFORM=OS_MACOSX
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
PLATFORM_SHARED_EXT=dylib
[ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/"
oIFS="$IFS"; IFS=.
set `uname -r`
IFS="$oIFS"
if [ "$1" -ge 13 ]; then
# assume clang compiler
COMMON_FLAGS="-mmacosx-version-min=10.8 -DOS_MACOSX -stdlib=libc++"
PLATFORM_LDFLAGS="-mmacosx-version-min=10.8"
else
COMMON_FLAGS="-fno-builtin-memcmp -DOS_MACOSX"
fi
PLATFORM_SHARED_EXT=
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
Linux)
PLATFORM=OS_LINUX
COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
PLATFORM_LDFLAGS="-pthread"
COMMON_FLAGS="-fno-builtin-memcmp -pthread -DOS_LINUX"
PLATFORM_LDFLAGS="-pthread -lrt"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
SunOS)
PLATFORM=OS_SOLARIS
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
PLATFORM_LIBS="-lpthread -lrt"
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
PLATFORM_LDFLAGS="-lpthread -lrt"
PLATFORM_SHARED_EXT=
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
FreeBSD)
CC=cc
CXX=c++
PLATFORM=OS_FREEBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
PLATFORM_LIBS="-lpthread"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
GNU/kFreeBSD)
PLATFORM=OS_KFREEBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_KFREEBSD"
PLATFORM_LIBS="-lpthread"
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
PLATFORM_LDFLAGS="-lpthread"
PORT_FILE=port/port_posix.cc
;;
NetBSD)
PLATFORM=OS_NETBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
PLATFORM_LIBS="-lpthread -lgcc_s"
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
PLATFORM_LDFLAGS="-lpthread -lgcc_s"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
OpenBSD)
PLATFORM=OS_OPENBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
PLATFORM_LDFLAGS="-pthread"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
DragonFly)
PLATFORM=OS_DRAGONFLYBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
PLATFORM_LIBS="-lpthread"
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
PLATFORM_LDFLAGS="-lpthread"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
;;
OS_ANDROID_CROSSCOMPILE)
PLATFORM=OS_ANDROID
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
CROSS_COMPILE=true
;;
HP-UX)
PLATFORM=OS_HPUX
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
PLATFORM_LDFLAGS="-pthread"
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
# man ld: +h internal_name
PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
;;
IOS)
PLATFORM=IOS
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
[ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
PORT_FILE=port/port_posix.cc
PORT_SSE_FILE=port/port_posix_sse.cc
PLATFORM_SHARED_EXT=
PLATFORM_SHARED_LDFLAGS=
PLATFORM_SHARED_CFLAGS=
PLATFORM_SHARED_VERSIONED=
;;
OS_WINDOWS_CROSSCOMPILE | NATIVE_WINDOWS)
PLATFORM=OS_WINDOWS
COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1"
PLATFORM_SOURCES="util/env_win.cc"
PLATFORM_LIBS="-lshlwapi"
PORT_FILE=port/port_win.cc
CROSS_COMPILE=true
;;
*)
@@ -182,78 +132,106 @@ esac
# except for the test and benchmark files. By default, find will output a list
# of all files matching either rule, so we need to append -print to make the
# prune take effect.
DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
if [ -f leveldb_ee/README.md ]; then
DIRS="util db table leveldb_ee"
else
DIRS="util db table leveldb_os"
fi
set -f # temporarily disable globbing so that our patterns aren't expanded
PRUNE_TEST="-name *test*.cc -prune"
PRUNE_BENCH="-name *_bench.cc -prune"
PRUNE_TOOL="-name leveldbutil.cc -prune"
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
TESTS=`find $DIRS -name '*_test.c*' -print | sort | tr "\n" " "`
set +f # re-enable globbing
# The sources consist of the portable files, plus the platform-specific port
# file.
echo "SOURCES=$PORTABLE_FILES $PORT_FILE $PORT_SSE_FILE" >> $OUTPUT
echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
echo "TEST_SOURCES=$TESTS" >>$OUTPUT
if [ "$CROSS_COMPILE" = "true" ]; then
# Cross-compiling; do not try any compilation tests.
true
else
CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
# If -std=c++0x works, use <atomic> as fallback for when memory barriers
# are not available.
$CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
#include <atomic>
# If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h.
$CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <cstdatomic>
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT"
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_CSTDATOMIC_PRESENT"
PLATFORM_CXXFLAGS="-std=c++0x"
else
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
fi
# Test whether Snappy library is installed
# http://code.google.com/p/snappy/
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <snappy.h>
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
if [ "$PLATFORM" = "OS_LINUX" ]; then
# Basho: switching to static snappy library to make tools more portable
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -Wl,-non_shared -lsnappy -Wl,-call_shared"
else
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
fi
fi
# Test whether tcmalloc is available
$CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null <<EOF
$CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
int main() {}
EOF
if [ "$?" = 0 ]; then
PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
fi
rm -f $CXXOUTPUT 2>/dev/null
# Test if gcc SSE 4.2 is supported
$CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -msse4.2 2>/dev/null <<EOF
int main() {}
EOF
if [ "$?" = 0 ]; then
PLATFORM_SSEFLAGS="-msse4.2"
fi
rm -f $CXXOUTPUT 2>/dev/null
fi
# Use the SSE 4.2 CRC32C intrinsics iff runtime checks indicate compiler supports them.
if [ -n "$PLATFORM_SSEFLAGS" ]; then
PLATFORM_SSEFLAGS="$PLATFORM_SSEFLAGS -DLEVELDB_PLATFORM_POSIX_SSE"
fi
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS $VERSION_FLAGS"
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS $VERSION_FLAGS"
echo "CC=$CC" >> $OUTPUT
echo "CXX=$CXX" >> $OUTPUT
echo "PLATFORM=$PLATFORM" >> $OUTPUT
echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT
echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
echo "PLATFORM_SSEFLAGS=$PLATFORM_SSEFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
#
# Basho extension to place -D variable in include/leveldb/ldb_config.h
#
LDB_CONFIG="include/leveldb/ldb_config.h"
# Delete existing output, if it exists
rm -f $LDB_CONFIG
write_config_h()
{
for param in $@
do
prefix=$(expr -- $param : "\(..\)")
if [ X$prefix = "X-D" ]
then
echo "" >>$LDB_CONFIG
echo "#ifndef $(expr -- $param : '..\(.*\)')" >>$LDB_CONFIG
echo " #define $(expr -- $param : '..\(.*\)')" >>$LDB_CONFIG
echo "#endif" >>$LDB_CONFIG
fi
done
}
echo "/** This file is generated by build_detect_platform." >$LDB_CONFIG
echo " * It saves the state of compile flags. This benefits the reuse" >>$LDB_CONFIG
echo " * of internal include files outside of a leveldb build." >>$LDB_CONFIG
echo " */" >>$LDB_CONFIG
write_config_h $COMMON_FLAGS
-118
View File
@@ -1,118 +0,0 @@
// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "leveldb/db.h"
#include "db/db_impl.h"
#include "leveldb/cache.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace leveldb {
class AutoCompactTest {
public:
std::string dbname_;
Cache* tiny_cache_;
Options options_;
DB* db_;
AutoCompactTest() {
dbname_ = test::TmpDir() + "/autocompact_test";
tiny_cache_ = NewLRUCache(100);
options_.block_cache = tiny_cache_;
DestroyDB(dbname_, options_);
options_.create_if_missing = true;
options_.compression = kNoCompression;
ASSERT_OK(DB::Open(options_, dbname_, &db_));
}
~AutoCompactTest() {
delete db_;
DestroyDB(dbname_, Options());
delete tiny_cache_;
}
std::string Key(int i) {
char buf[100];
snprintf(buf, sizeof(buf), "key%06d", i);
return std::string(buf);
}
uint64_t Size(const Slice& start, const Slice& limit) {
Range r(start, limit);
uint64_t size;
db_->GetApproximateSizes(&r, 1, &size);
return size;
}
void DoReads(int n);
};
static const int kValueSize = 200 * 1024;
static const int kTotalSize = 100 * 1024 * 1024;
static const int kCount = kTotalSize / kValueSize;
// Read through the first n keys repeatedly and check that they get
// compacted (verified by checking the size of the key space).
void AutoCompactTest::DoReads(int n) {
std::string value(kValueSize, 'x');
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
// Fill database
for (int i = 0; i < kCount; i++) {
ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
}
ASSERT_OK(dbi->TEST_CompactMemTable());
// Delete everything
for (int i = 0; i < kCount; i++) {
ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
}
ASSERT_OK(dbi->TEST_CompactMemTable());
// Get initial measurement of the space we will be reading.
const int64_t initial_size = Size(Key(0), Key(n));
const int64_t initial_other_size = Size(Key(n), Key(kCount));
// Read until size drops significantly.
std::string limit_key = Key(n);
for (int read = 0; true; read++) {
ASSERT_LT(read, 100) << "Taking too long to compact";
Iterator* iter = db_->NewIterator(ReadOptions());
for (iter->SeekToFirst();
iter->Valid() && iter->key().ToString() < limit_key;
iter->Next()) {
// Drop data
}
delete iter;
// Wait a little bit to allow any triggered compactions to complete.
Env::Default()->SleepForMicroseconds(1000000);
uint64_t size = Size(Key(0), Key(n));
fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
if (size <= initial_size/10) {
break;
}
}
// Verify that the size of the key space not touched by the reads
// is pretty much unchanged.
const int64_t final_other_size = Size(Key(n), Key(kCount));
ASSERT_LE(final_other_size, initial_other_size + 1048576);
ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
}
TEST(AutoCompactTest, ReadAll) {
DoReads(kCount);
}
TEST(AutoCompactTest, ReadHalf) {
DoReads(kCount/2);
}
} // namespace leveldb
int main(int argc, char** argv) {
return leveldb::test::RunAllTests();
}
+52 -6
View File
@@ -2,12 +2,16 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include "db/builder.h"
#include "db/filename.h"
#include "db/dbformat.h"
#include "db/table_cache.h"
#include "db/version_edit.h"
#include "db/version_set.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/iterator.h"
@@ -17,27 +21,51 @@ namespace leveldb {
Status BuildTable(const std::string& dbname,
Env* env,
const Options& options,
const Comparator * user_comparator,
TableCache* table_cache,
Iterator* iter,
FileMetaData* meta) {
FileMetaData* meta,
SequenceNumber smallest_snapshot) {
Status s;
size_t keys_seen, keys_retired;
keys_seen=0;
keys_retired=0;
meta->file_size = 0;
iter->SeekToFirst();
std::string fname = TableFileName(dbname, meta->number);
KeyRetirement retire(user_comparator, smallest_snapshot, &options);
std::string fname = TableFileName(options, meta->number, meta->level);
if (iter->Valid()) {
WritableFile* file;
s = env->NewWritableFile(fname, &file);
s = env->NewWritableFile(fname, &file,
env->RecoveryMmapSize(&options));
if (!s.ok()) {
return s;
}
// tune fadvise to keep all of this lower level file in page cache
// (compaction of unsorted files causes severe cache misses)
file->SetMetadataOffset(1);
TableBuilder* builder = new TableBuilder(options, file);
meta->smallest.DecodeFrom(iter->key());
for (; iter->Valid(); iter->Next()) {
++keys_seen;
Slice key = iter->key();
meta->largest.DecodeFrom(key);
builder->Add(key, iter->value());
if (!retire(key))
{
meta->largest.DecodeFrom(key);
builder->Add(key, iter->value());
++meta->num_entries;
} // if
else
{
++keys_retired;
} // else
}
// Finish and check for builder errors
@@ -45,6 +73,9 @@ Status BuildTable(const std::string& dbname,
s = builder->Finish();
if (s.ok()) {
meta->file_size = builder->FileSize();
meta->exp_write_low = builder->GetExpiryWriteLow();
meta->exp_write_high = builder->GetExpiryWriteHigh();
meta->exp_explicit_high = builder->GetExpiryExplicitHigh();
assert(meta->file_size > 0);
}
} else {
@@ -64,10 +95,20 @@ Status BuildTable(const std::string& dbname,
if (s.ok()) {
// Verify that the table is usable
Table * table_ptr;
Iterator* it = table_cache->NewIterator(ReadOptions(),
meta->number,
meta->file_size);
meta->file_size,
meta->level,
&table_ptr);
s = it->status();
// Riak specific: bloom filter is no longer read by default,
// force read on highly used overlapped table files
if (s.ok() && VersionSet::IsLevelOverlapped(meta->level))
table_ptr->ReadFilter();
// table_ptr is owned by it and therefore invalidated by this delete
delete it;
}
}
@@ -79,6 +120,11 @@ Status BuildTable(const std::string& dbname,
if (s.ok() && meta->file_size > 0) {
// Keep it
if (0!=keys_retired)
{
Log(options.info_log, "Level-0 table #%" PRIu64 ": %zd keys seen, %zd keys retired, %zd keys expired",
meta->number, keys_seen, retire.GetDroppedCount(), retire.GetExpiredCount());
} // if
} else {
env->DeleteFile(fname);
}
+4 -1
View File
@@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_DB_BUILDER_H_
#include "leveldb/status.h"
#include "db/dbformat.h"
namespace leveldb {
@@ -25,9 +26,11 @@ class VersionEdit;
extern Status BuildTable(const std::string& dbname,
Env* env,
const Options& options,
const Comparator * user_comparator,
TableCache* table_cache,
Iterator* iter,
FileMetaData* meta);
FileMetaData* meta,
SequenceNumber smallest_snapshot);
} // namespace leveldb
+77 -8
View File
@@ -6,6 +6,7 @@
#include <stdlib.h>
#include <unistd.h>
#include <stdint.h>
#include "leveldb/cache.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
@@ -40,6 +41,8 @@ using leveldb::Status;
using leveldb::WritableFile;
using leveldb::WriteBatch;
using leveldb::WriteOptions;
using leveldb::KeyMetaData;
using leveldb::ValueType;
extern "C" {
@@ -49,6 +52,7 @@ struct leveldb_writebatch_t { WriteBatch rep; };
struct leveldb_snapshot_t { const Snapshot* rep; };
struct leveldb_readoptions_t { ReadOptions rep; };
struct leveldb_writeoptions_t { WriteOptions rep; };
struct leveldb_keymetadata_t { KeyMetaData rep; };
struct leveldb_options_t { Options rep; };
struct leveldb_cache_t { Cache* rep; };
struct leveldb_seqfile_t { SequentialFile* rep; };
@@ -173,8 +177,19 @@ void leveldb_put(
const char* key, size_t keylen,
const char* val, size_t vallen,
char** errptr) {
return(leveldb_put2(db, options, key, keylen, val, vallen, errptr, NULL));
}
void leveldb_put2(
leveldb_t* db,
const leveldb_writeoptions_t* options,
const char* key, size_t keylen,
const char* val, size_t vallen,
char** errptr,
const leveldb_keymetadata_t * metadata) {
SaveError(errptr,
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen),
(NULL==metadata ? NULL : &metadata->rep)));
}
void leveldb_delete(
@@ -200,9 +215,21 @@ char* leveldb_get(
const char* key, size_t keylen,
size_t* vallen,
char** errptr) {
return(leveldb_get2(db, options, key, keylen, vallen, errptr, NULL));
}
char* leveldb_get2(
leveldb_t* db,
const leveldb_readoptions_t* options,
const char* key, size_t keylen,
size_t* vallen,
char** errptr,
leveldb_keymetadata_t * metadata) {
char* result = NULL;
std::string tmp;
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp,
(NULL==metadata ? NULL : &metadata->rep));
if (s.ok()) {
*vallen = tmp.size();
result = CopyString(tmp);
@@ -330,6 +357,15 @@ const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) {
return s.data();
}
const void leveldb_iter_keymetadata(const leveldb_iterator_t* iter,
leveldb_keymetadata_t * meta)
{
if (NULL!=iter && NULL!=meta)
{
meta->rep=iter->rep->keymetadata();
} // if
}
void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) {
SaveError(errptr, iter->rep->status());
}
@@ -350,7 +386,16 @@ void leveldb_writebatch_put(
leveldb_writebatch_t* b,
const char* key, size_t klen,
const char* val, size_t vlen) {
b->rep.Put(Slice(key, klen), Slice(val, vlen));
leveldb_writebatch_put2(b, key, klen, val, vlen,NULL);
}
void leveldb_writebatch_put2(
leveldb_writebatch_t* b,
const char* key, size_t klen,
const char* val, size_t vlen,
const leveldb_keymetadata_t * metadata) {
b->rep.Put(Slice(key, klen), Slice(val, vlen),
(NULL==metadata ? NULL : &metadata->rep));
}
void leveldb_writebatch_delete(
@@ -362,15 +407,20 @@ void leveldb_writebatch_delete(
void leveldb_writebatch_iterate(
leveldb_writebatch_t* b,
void* state,
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen,
const int & type, const uint64_t & expiry),
void (*deleted)(void*, const char* k, size_t klen)) {
class H : public WriteBatch::Handler {
public:
void* state_;
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen,
const int & type, const uint64_t & expiry);
void (*deleted_)(void*, const char* k, size_t klen);
virtual void Put(const Slice& key, const Slice& value) {
(*put_)(state_, key.data(), key.size(), value.data(), value.size());
virtual void Put(const Slice& key, const Slice& value,
const leveldb::ValueType & type,
const leveldb::ExpiryTimeMicros & expiry)
{
(*put_)(state_, key.data(), key.size(), value.data(), value.size(), (int)type, (uint64_t)expiry);
}
virtual void Delete(const Slice& key) {
(*deleted_)(state_, key.data(), key.size());
@@ -418,6 +468,11 @@ void leveldb_options_set_paranoid_checks(
opt->rep.paranoid_checks = v;
}
void leveldb_options_set_verify_compactions(
leveldb_options_t* opt, unsigned char v) {
opt->rep.verify_compactions = v;
}
void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
opt->rep.env = (env ? env->rep : NULL);
}
@@ -450,6 +505,10 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
opt->rep.compression = static_cast<CompressionType>(t);
}
void leveldb_options_set_total_leveldb_mem(leveldb_options_t* opt, size_t s) {
opt->rep.total_leveldb_mem = s;
}
leveldb_comparator_t* leveldb_comparator_create(
void* state,
void (*destructor)(void*),
@@ -580,8 +639,18 @@ void leveldb_env_destroy(leveldb_env_t* env) {
delete env;
}
void leveldb_env_shutdown() {
Env::Shutdown();
}
/**
* CAUTION: this call is only for char * objects returned by
* functions like leveldb_get and leveldb_property_value.
* Also used to release errptr strings.
*/
void leveldb_free(void* ptr) {
free(ptr);
if (NULL!=ptr)
free(ptr);
}
int leveldb_major_version() {
+225 -12
View File
@@ -3,6 +3,8 @@
found in the LICENSE file. See the AUTHORS file for names of contributors. */
#include "leveldb/c.h"
#include "leveldb/options.h"
#include "port/port.h"
#include <stddef.h>
#include <stdio.h>
@@ -11,8 +13,13 @@
#include <sys/types.h>
#include <unistd.h>
using leveldb::ValueType;
struct leveldb_keymetadata_t { leveldb::KeyMetaData rep; };
const char* phase = "";
static char dbname[200];
static leveldb::ExpiryTimeMicros gStartTime;
static void StartPhase(const char* name) {
fprintf(stderr, "=== Test %s\n", name);
@@ -33,7 +40,7 @@ static const char* GetTempDir(void) {
}
#define CheckCondition(cond) \
if (!(cond)) { \
if (!(cond)) { \
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
abort(); \
}
@@ -49,7 +56,7 @@ static void CheckEqual(const char* expected, const char* v, size_t n) {
fprintf(stderr, "%s: expected '%s', got '%s'\n",
phase,
(expected ? expected : "(null)"),
(v ? v : "(null"));
(v ? v : "(null)"));
abort();
}
}
@@ -112,6 +119,117 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
(*state)++;
}
// (expiry enabled)
static void CheckGet2(
leveldb_t* db,
const leveldb_readoptions_t* options,
const char* key,
const char* expected,
ValueType type,
uint64_t expiry) {
char* err = NULL;
size_t val_len;
char* val;
leveldb_keymetadata_t meta;
val = leveldb_get2(db, options, key, strlen(key), &val_len, &err, &meta);
CheckNoError(err);
CheckEqual(expected, val, val_len);
CheckCondition(type==meta.rep.m_Type);
if (0==expiry && leveldb::kTypeValueWriteTime==type)
{
leveldb::ExpiryTimeMicros now=leveldb::port::TimeMicros();
CheckCondition(gStartTime<=meta.rep.m_Expiry && meta.rep.m_Expiry<=now);
} // if
else
{CheckCondition(expiry==meta.rep.m_Expiry);}
Free(&val);
}
// (expiry enabled)
static void CheckIter2(leveldb_iterator_t* iter,
const char* key, const char* val,
const leveldb::KeyMetaData & meta) {
size_t len;
const char* str;
leveldb_keymetadata_t it_meta;
str = leveldb_iter_key(iter, &len);
CheckEqual(key, str, len);
str = leveldb_iter_value(iter, &len);
CheckEqual(val, str, len);
leveldb_iter_keymetadata(iter, &it_meta);
CheckCondition(meta.m_Type==it_meta.rep.m_Type);
if (0==meta.m_Expiry && leveldb::kTypeValueWriteTime==meta.m_Type)
{
leveldb::ExpiryTimeMicros now=leveldb::port::TimeMicros();
CheckCondition(gStartTime<=it_meta.rep.m_Expiry && it_meta.rep.m_Expiry<=now);
} // if
else
{CheckCondition(meta.m_Expiry==it_meta.rep.m_Expiry);}
}
// Callback from leveldb_writebatch_iterate()
// (expiry enabled)
struct CheckPut2Data
{
const char * m_Key;
const char * m_Value;
ValueType m_Type;
uint64_t m_Expiry;
};
static struct CheckPut2Data gCheckPut2Data[]=
{
{"foo","hello_put2",leveldb::kTypeValue,0},
{"box","c_put2",leveldb::kTypeValue,0},
{"disney","cartoon_put2",leveldb::kTypeValueWriteTime, 0},
{"money","lotsof_put2",leveldb::kTypeValueWriteTime, 9988776655},
{"time","ismoney_put2",leveldb::kTypeValueExplicitExpiry, 221199887766}
};
static struct CheckPut2Data gCheckPut2ItrData[]=
{
{"bar","b",leveldb::kTypeValue,0},
{"box","c",leveldb::kTypeValue,0},
{"bar","",leveldb::kTypeDeletion,0},
{"mom","texas",leveldb::kTypeValueWriteTime,0},
{"dad","poland",leveldb::kTypeValueExplicitExpiry,22446688}
};
static void CheckPut2(void* ptr,
const char* k, size_t klen,
const char* v, size_t vlen,
const int & type_int,
const uint64_t & expiry) {
int* state = (int*) ptr;
CheckCondition(*state < (sizeof(gCheckPut2ItrData)/sizeof(gCheckPut2ItrData[0])));
struct CheckPut2Data * test;
test=&gCheckPut2ItrData[*state];
CheckEqual(test->m_Key, k, klen);
CheckEqual(test->m_Value, v, vlen);
CheckCondition((int)test->m_Type==type_int);
if (leveldb::kTypeValueWriteTime!=test->m_Type)
{CheckCondition((uint64_t)test->m_Expiry==expiry);}
(*state)++;
}
// Callback from leveldb_writebatch_iterate()
// (expiry enabled)
static void CheckDel2(void* ptr, const char* k, size_t klen) {
int* state = (int*) ptr;
CheckCondition(*state < (sizeof(gCheckPut2ItrData)/sizeof(gCheckPut2ItrData[0])));
struct CheckPut2Data * test;
test=&gCheckPut2ItrData[*state];
CheckEqual(test->m_Key, k, klen);
(*state)++;
}
static void CmpDestroy(void* arg) { }
static int CmpCompare(void* arg, const char* a, size_t alen,
@@ -141,7 +259,7 @@ static char* FilterCreate(
int num_keys,
size_t* filter_length) {
*filter_length = 4;
char* result = malloc(4);
char* result = (char*)malloc(4);
memcpy(result, "fake", 4);
return result;
}
@@ -167,6 +285,7 @@ int main(int argc, char** argv) {
CheckCondition(leveldb_major_version() >= 1);
CheckCondition(leveldb_minor_version() >= 1);
gStartTime=leveldb::port::TimeMicros();
snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%d",
@@ -207,12 +326,6 @@ int main(int argc, char** argv) {
CheckCondition(err != NULL);
Free(&err);
StartPhase("leveldb_free");
db = leveldb_open(options, dbname, &err);
CheckCondition(err != NULL);
leveldb_free(err);
err = NULL;
StartPhase("open");
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, dbname, &err);
@@ -234,42 +347,74 @@ int main(int argc, char** argv) {
StartPhase("writebatch");
{
leveldb_keymetadata_t meta;
leveldb_writebatch_t* wb = leveldb_writebatch_create();
leveldb_writebatch_put(wb, "foo", 3, "a", 1);
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
leveldb_writebatch_delete(wb, "bar", 3);
meta.rep.m_Type=leveldb::kTypeValueWriteTime;
meta.rep.m_Expiry=0;
leveldb_writebatch_put2(wb, "mom", 3, "texas", 5, &meta);
meta.rep.m_Type=leveldb::kTypeValueExplicitExpiry;
meta.rep.m_Expiry=22446688;
leveldb_writebatch_put2(wb, "dad", 3, "poland", 6, &meta);
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
CheckGet2(db, roptions, "dad", "poland", leveldb::kTypeValueExplicitExpiry, 22446688);
CheckGet2(db, roptions, "mom", "texas", leveldb::kTypeValueWriteTime, 0);
int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
leveldb_writebatch_iterate(wb, &pos, CheckPut2, CheckDel2);
CheckCondition(pos == 5);
leveldb_writebatch_destroy(wb);
}
// reminder: keymetadata not supported on backward iteration
StartPhase("iter");
{
leveldb::KeyMetaData meta;
leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_first(iter);
CheckCondition(leveldb_iter_valid(iter));
CheckIter(iter, "box", "c");
meta.m_Type=leveldb::kTypeValue;
meta.m_Expiry=0;
CheckIter2(iter, "box", "c", meta);
meta.m_Type=leveldb::kTypeValueExplicitExpiry;
meta.m_Expiry=22446688;
leveldb_iter_next(iter);
CheckIter2(iter, "dad", "poland", meta);
leveldb_iter_next(iter);
CheckIter(iter, "foo", "hello");
leveldb_iter_prev(iter);
CheckIter(iter, "dad", "poland");
leveldb_iter_prev(iter);
CheckIter(iter, "box", "c");
leveldb_iter_prev(iter);
CheckCondition(!leveldb_iter_valid(iter));
leveldb_iter_seek_to_last(iter);
CheckIter(iter, "foo", "hello");
CheckIter(iter, "mom", "texas");
leveldb_iter_seek(iter, "b", 1);
CheckIter(iter, "box", "c");
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
meta.m_Type=leveldb::kTypeValue;
meta.m_Expiry=0;
CheckIter2(iter, "box", "c", meta);
leveldb_iter_seek(iter, "m", 1);
meta.m_Type=leveldb::kTypeValueWriteTime;
meta.m_Expiry=0;
CheckIter2(iter, "mom", "texas", meta);
leveldb_iter_get_error(iter, &err);
CheckNoError(err);
leveldb_iter_destroy(iter);
}
@@ -335,6 +480,70 @@ int main(int argc, char** argv) {
leveldb_options_set_error_if_exists(options, 1);
}
StartPhase("put expiry");
{
leveldb_keymetadata_t meta;
int loop, count;
count = sizeof(gCheckPut2Data) / sizeof(gCheckPut2Data[0]);
for (loop=0; loop<count; ++loop)
{
size_t klen, vlen;
leveldb_keymetadata_t meta;
struct CheckPut2Data * test;
test=&gCheckPut2Data[loop];
klen=strlen(test->m_Key);
vlen=strlen(test->m_Value);
meta.rep.m_Type=test->m_Type;
meta.rep.m_Expiry=test->m_Expiry;
leveldb_put2(db, woptions, test->m_Key, klen,
test->m_Value, vlen, &err,
&meta);
CheckNoError(err);
} // for
// testing memtable right now
for (loop=0; loop<count; ++loop)
{
size_t klen, vlen;
leveldb_keymetadata_t meta;
struct CheckPut2Data * test;
test=&gCheckPut2Data[loop];
klen=strlen(test->m_Key);
vlen=strlen(test->m_Value);
CheckGet2(db, roptions, test->m_Key, test->m_Value,
test->m_Type, test->m_Expiry);
} // for
// close and open to force memory table into .sst upon open
leveldb_close(db);
leveldb_options_set_error_if_exists(options, 0);
db = leveldb_open(options, dbname, &err);
CheckNoError(err);
// now testing get from a level-0 .sst file
for (loop=0; loop<count; ++loop)
{
size_t klen, vlen;
leveldb_keymetadata_t meta;
struct CheckPut2Data * test;
test=&gCheckPut2Data[loop];
klen=strlen(test->m_Key);
vlen=strlen(test->m_Value);
CheckGet2(db, roptions, test->m_Key, test->m_Value,
test->m_Type, test->m_Expiry);
} // for
}
//
// This screws up "options" for real database work. execute last.
StartPhase("filter");
for (run = 0; run < 2; run++) {
// First run uses custom filter, second run uses bloom filter
@@ -376,6 +585,8 @@ int main(int argc, char** argv) {
leveldb_filterpolicy_destroy(policy);
}
StartPhase("cleanup");
leveldb_close(db);
leveldb_options_destroy(options);
@@ -386,5 +597,7 @@ int main(int argc, char** argv) {
leveldb_env_destroy(env);
fprintf(stderr, "PASS\n");
leveldb_env_shutdown();
return 0;
}
+63 -58
View File
@@ -35,8 +35,8 @@ class CorruptionTest {
CorruptionTest() {
tiny_cache_ = NewLRUCache(100);
options_.env = &env_;
options_.block_cache = tiny_cache_;
dbname_ = test::TmpDir() + "/corruption_test";
dbname_ = test::TmpDir() + "/db_test";
dbname_ = MakeTieredDbname(dbname_, options_);
DestroyDB(dbname_, options_);
db_ = NULL;
@@ -51,14 +51,17 @@ class CorruptionTest {
delete tiny_cache_;
}
Status TryReopen() {
Status TryReopen(Options* options = NULL) {
delete db_;
db_ = NULL;
return DB::Open(options_, dbname_, &db_);
Options opt = (options ? *options : options_);
opt.env = &env_;
opt.block_cache = tiny_cache_;
return DB::Open(opt, dbname_, &db_);
}
void Reopen() {
ASSERT_OK(TryReopen());
void Reopen(Options* options = NULL) {
ASSERT_OK(TryReopen(options));
}
void RepairDB() {
@@ -75,13 +78,7 @@ class CorruptionTest {
Slice key = Key(i, &key_space);
batch.Clear();
batch.Put(key, Value(i, &value_space));
WriteOptions options;
// Corrupt() doesn't work without this sync on windows; stat reports 0 for
// the file size.
if (i == n - 1) {
options.sync = true;
}
ASSERT_OK(db_->Write(options, &batch));
ASSERT_OK(db_->Write(WriteOptions(), &batch));
}
}
@@ -96,10 +93,6 @@ class CorruptionTest {
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
uint64_t key;
Slice in(iter->key());
if (in == "" || in == "~") {
// Ignore boundary keys.
continue;
}
if (!ConsumeDecimalNumber(&in, &key) ||
!in.empty() ||
key < next_expected) {
@@ -123,19 +116,26 @@ class CorruptionTest {
ASSERT_GE(max_expected, correct);
}
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt, int level=0) {
// Pick file to corrupt
std::vector<std::string> filenames;
ASSERT_OK(env_.GetChildren(dbname_, &filenames));
std::string dirname;
if (leveldb::kTableFile!=filetype)
dirname=dbname_;
else
dirname=MakeDirName2(options_, level, "sst");
ASSERT_OK(env_.GetChildren(dirname, &filenames));
uint64_t number;
FileType type;
std::string fname;
int picked_number = -1;
for (size_t i = 0; i < filenames.size(); i++) {
for (int i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) &&
type == filetype &&
int(number) > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i];
fname = dirname + "/" + filenames[i];
picked_number = number;
}
}
@@ -222,12 +222,14 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
const int num = 3 + (Options().write_buffer_size / kValueSize);
std::string value_storage;
Status s;
for (int i = 0; s.ok() && i < num; i++) {
for (int i = 0;
s.ok() && i < num && 0==env_.num_writable_file_errors_;
i++) {
WriteBatch batch;
batch.Put("a", Value(100, &value_storage));
s = db_->Write(WriteOptions(), &batch);
}
ASSERT_TRUE(!s.ok());
// ASSERT_TRUE(!s.ok()); Background write thread will never report this
ASSERT_GE(env_.num_writable_file_errors_, 1);
env_.writable_file_error_ = false;
Reopen();
@@ -240,34 +242,18 @@ TEST(CorruptionTest, TableFile) {
dbi->TEST_CompactRange(0, NULL, NULL);
dbi->TEST_CompactRange(1, NULL, NULL);
Corrupt(kTableFile, 100, 1);
Check(90, 99);
}
TEST(CorruptionTest, TableFileRepair) {
options_.block_size = 2 * kValueSize; // Limit scope of corruption
options_.paranoid_checks = true;
Reopen();
Build(100);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
dbi->TEST_CompactRange(0, NULL, NULL);
dbi->TEST_CompactRange(1, NULL, NULL);
Corrupt(kTableFile, 100, 1);
RepairDB();
Reopen();
Corrupt(kTableFile, 100, 1, config::kMaxMemCompactLevel);
Check(95, 99);
}
TEST(CorruptionTest, TableFileIndexData) {
Build(10000); // Enough to build multiple Tables
Build(100000); // Enough to build multiple Tables
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
Corrupt(kTableFile, -2000, 500);
Corrupt(kTableFile, -2000, 500, config::kMaxMemCompactLevel);
Reopen();
Check(5000, 9999);
Check(50000, 99999);
}
TEST(CorruptionTest, MissingDescriptor) {
@@ -319,10 +305,10 @@ TEST(CorruptionTest, CompactionInputError) {
Build(10);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
const int last = config::kMaxMemCompactLevel;
const int last = config::kMaxMemCompactLevel; // Riak does not "move" files
ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
Corrupt(kTableFile, 100, 1);
Corrupt(kTableFile, 100, 1, last);
Check(5, 9);
// Force compactions by writing lots of values
@@ -331,31 +317,50 @@ TEST(CorruptionTest, CompactionInputError) {
}
TEST(CorruptionTest, CompactionInputErrorParanoid) {
options_.paranoid_checks = true;
options_.write_buffer_size = 512 << 10;
Reopen();
Options options;
options.paranoid_checks = true;
options.write_buffer_size = 1048576;
Reopen(&options);
int current_corruption=Property("leveldb.ReadBlockError");
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
// Make multiple inputs so we need to compact.
for (int i = 0; i < 2; i++) {
Build(10);
// Fill levels >= 1 so memtable compaction outputs to level 1
// matthewv 1/10/14 - what does "levels" have to do with this,
// switching to compaction trigger.
// 7/10/14 - compaction starts between 4 and 6 files ... assume 4 and 1 move
// (will make a new, descriptive constant for 4)
for (int level = Property("leveldb.num-files-at-level0")+1;
level < config::kL0_GroomingTrigger; level++) {
dbi->Put(WriteOptions(), "", "begin");
dbi->Put(WriteOptions(), "~", "end");
dbi->TEST_CompactMemTable();
Corrupt(kTableFile, 100, 1);
env_.SleepForMicroseconds(100000);
}
dbi->CompactRange(NULL, NULL);
// Write must fail because of corrupted table
Build(10);
dbi->TEST_CompactMemTable();
ASSERT_TRUE(1 < Property("leveldb.num-files-at-level0"));
Corrupt(kTableFile, 100, 1, 0);
Check(5, 9);
// Write must eventually fail because of corrupted table
Status s;
std::string tmp1, tmp2;
Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
for (int i = 0; i < 10000 && s.ok(); i++) {
s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
}
if (s.ok())
ASSERT_NE(current_corruption, Property("leveldb.ReadBlockError")) << "no ReadBlockError seen";
else
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
}
TEST(CorruptionTest, UnrelatedKeys) {
Build(10);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
Corrupt(kTableFile, 100, 1);
Corrupt(kTableFile, 100, 1, config::kMaxMemCompactLevel);
std::string tmp1, tmp2;
ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
+46 -61
View File
@@ -33,7 +33,6 @@
// readmissing -- read N missing keys in random order
// readhot -- read N times in random order from 1% section of DB
// seekrandom -- N random seeks
// open -- cost of opening a DB
// crc32c -- repeated crc32c of 4K of data
// acquireload -- load N*1000 times
// Meta operations:
@@ -84,14 +83,6 @@ static bool FLAGS_histogram = false;
// (initialized to default value by "main")
static int FLAGS_write_buffer_size = 0;
// Number of bytes written to each file.
// (initialized to default value by "main")
static int FLAGS_max_file_size = 0;
// Approximate size of user data packed per block (before compression.
// (initialized to default value by "main")
static int FLAGS_block_size = 0;
// Number of bytes to use as a cache of uncompressed data.
// Negative means use default settings.
static int FLAGS_cache_size = -1;
@@ -103,21 +94,26 @@ static int FLAGS_open_files = 0;
// Negative means use default settings.
static int FLAGS_bloom_bits = -1;
// Riak bloom adaptation
static int FLAGS_bloom2_bits = -1;
// Riak param for total memory allocation (flex_cache)
static uint64_t FLAGS_leveldb_memory = -1;
// Riak param for compression setting
static int FLAGS_compression = 2;
// If true, do not destroy the existing database. If you set this
// flag and also specify a benchmark that wants a fresh database, that
// benchmark will fail.
static bool FLAGS_use_existing_db = false;
// If true, reuse existing log/MANIFEST files when re-opening a database.
static bool FLAGS_reuse_logs = false;
// Use the db with the following name.
static const char* FLAGS_db = NULL;
namespace leveldb {
namespace {
leveldb::Env* g_env = NULL;
// Helper for quickly generating random data.
class RandomGenerator {
@@ -141,7 +137,7 @@ class RandomGenerator {
pos_ = 0;
}
Slice Generate(size_t len) {
Slice Generate(int len) {
if (pos_ + len > data_.size()) {
pos_ = 0;
assert(len < data_.size());
@@ -151,19 +147,17 @@ class RandomGenerator {
}
};
#if defined(__linux)
static Slice TrimSpace(Slice s) {
size_t start = 0;
int start = 0;
while (start < s.size() && isspace(s[start])) {
start++;
}
size_t limit = s.size();
int limit = s.size();
while (limit > start && isspace(s[limit-1])) {
limit--;
}
return Slice(s.data() + start, limit - start);
}
#endif
static void AppendWithSpace(std::string* str, Slice msg) {
if (msg.empty()) return;
@@ -195,7 +189,7 @@ class Stats {
done_ = 0;
bytes_ = 0;
seconds_ = 0;
start_ = g_env->NowMicros();
start_ = Env::Default()->NowMicros();
finish_ = start_;
message_.clear();
}
@@ -213,7 +207,7 @@ class Stats {
}
void Stop() {
finish_ = g_env->NowMicros();
finish_ = Env::Default()->NowMicros();
seconds_ = (finish_ - start_) * 1e-6;
}
@@ -223,7 +217,7 @@ class Stats {
void FinishedSingleOp() {
if (FLAGS_histogram) {
double now = g_env->NowMicros();
double now = Env::Default()->NowMicros();
double micros = now - last_op_finish_;
hist_.Add(micros);
if (micros > 20000) {
@@ -405,7 +399,7 @@ class Benchmark {
: cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
filter_policy_(FLAGS_bloom_bits >= 0
? NewBloomFilterPolicy(FLAGS_bloom_bits)
: NULL),
: (FLAGS_bloom2_bits >=0 ? NewBloomFilterPolicy2(FLAGS_bloom2_bits) : NULL)),
db_(NULL),
num_(FLAGS_num),
value_size_(FLAGS_value_size),
@@ -413,10 +407,10 @@ class Benchmark {
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
heap_counter_(0) {
std::vector<std::string> files;
g_env->GetChildren(FLAGS_db, &files);
for (size_t i = 0; i < files.size(); i++) {
Env::Default()->GetChildren(FLAGS_db, &files);
for (int i = 0; i < files.size(); i++) {
if (Slice(files[i]).starts_with("heap-")) {
g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
}
}
if (!FLAGS_use_existing_db) {
@@ -446,7 +440,7 @@ class Benchmark {
benchmarks = sep + 1;
}
// Reset parameters that may be overridden below
// Reset parameters that may be overriddden bwlow
num_ = FLAGS_num;
reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
value_size_ = FLAGS_value_size;
@@ -457,11 +451,7 @@ class Benchmark {
bool fresh_db = false;
int num_threads = FLAGS_threads;
if (name == Slice("open")) {
method = &Benchmark::OpenBench;
num_ /= 10000;
if (num_ < 1) num_ = 1;
} else if (name == Slice("fillseq")) {
if (name == Slice("fillseq")) {
fresh_db = true;
method = &Benchmark::WriteSeq;
} else if (name == Slice("fillbatch")) {
@@ -553,6 +543,7 @@ class Benchmark {
SharedState* shared;
ThreadState* thread;
void (Benchmark::*method)(ThreadState*);
pthread_t thread_id;
};
static void ThreadBody(void* v) {
@@ -598,7 +589,8 @@ class Benchmark {
arg[i].shared = &shared;
arg[i].thread = new ThreadState(i);
arg[i].thread->shared = &shared;
g_env->StartThread(ThreadBody, &arg[i]);
arg[i].thread_id=Env::Default()->StartThread(ThreadBody, &arg[i]);
pthread_detach(arg[i].thread_id);
}
shared.mu.Lock();
@@ -709,15 +701,12 @@ class Benchmark {
void Open() {
assert(db_ == NULL);
Options options;
options.env = g_env;
options.create_if_missing = !FLAGS_use_existing_db;
options.block_cache = cache_;
options.write_buffer_size = FLAGS_write_buffer_size;
options.max_file_size = FLAGS_max_file_size;
options.block_size = FLAGS_block_size;
options.max_open_files = FLAGS_open_files;
options.filter_policy = filter_policy_;
options.reuse_logs = FLAGS_reuse_logs;
options.compression = (leveldb::CompressionType)FLAGS_compression;
options.total_leveldb_mem = FLAGS_leveldb_memory;
Status s = DB::Open(options, FLAGS_db, &db_);
if (!s.ok()) {
fprintf(stderr, "open error: %s\n", s.ToString().c_str());
@@ -725,14 +714,6 @@ class Benchmark {
}
}
void OpenBench(ThreadState* thread) {
for (int i = 0; i < num_; i++) {
delete db_;
Open();
thread->stats.FinishedSingleOp();
}
}
void WriteSeq(ThreadState* thread) {
DoWrite(thread, true);
}
@@ -842,6 +823,7 @@ class Benchmark {
void SeekRandom(ThreadState* thread) {
ReadOptions options;
std::string value;
int found = 0;
for (int i = 0; i < reads_; i++) {
Iterator* iter = db_->NewIterator(options);
@@ -937,7 +919,7 @@ class Benchmark {
char fname[100];
snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
WritableFile* file;
Status s = g_env->NewWritableFile(fname, &file);
Status s = Env::Default()->NewWritableFile(fname, &file, 2<<20);
if (!s.ok()) {
fprintf(stderr, "%s\n", s.ToString().c_str());
return;
@@ -946,7 +928,7 @@ class Benchmark {
delete file;
if (!ok) {
fprintf(stderr, "heap profiling not supported\n");
g_env->DeleteFile(fname);
Env::Default()->DeleteFile(fname);
}
}
};
@@ -955,14 +937,14 @@ class Benchmark {
int main(int argc, char** argv) {
FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
FLAGS_max_file_size = leveldb::Options().max_file_size;
FLAGS_block_size = leveldb::Options().block_size;
FLAGS_open_files = leveldb::Options().max_open_files;
FLAGS_leveldb_memory = 25000000000LL;
std::string default_db_path;
for (int i = 1; i < argc; i++) {
double d;
int n;
uint64_t u;
char junk;
if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
@@ -974,9 +956,6 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_use_existing_db = n;
} else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_reuse_logs = n;
} else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
FLAGS_num = n;
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
@@ -987,16 +966,18 @@ int main(int argc, char** argv) {
FLAGS_value_size = n;
} else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
FLAGS_write_buffer_size = n;
} else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) {
FLAGS_max_file_size = n;
} else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
FLAGS_block_size = n;
} else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
FLAGS_cache_size = n;
} else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
FLAGS_bloom_bits = n;
} else if (sscanf(argv[i], "--bloom_bits2=%d%c", &n, &junk) == 1) {
FLAGS_bloom2_bits = n;
} else if (sscanf(argv[i], "--leveldb_memory=%d%c", &n, &junk) == 1) {
FLAGS_leveldb_memory = n * 1024 * 1024LL;
} else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
FLAGS_open_files = n;
} else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1) {
FLAGS_compression = n;
} else if (strncmp(argv[i], "--db=", 5) == 0) {
FLAGS_db = argv[i] + 5;
} else {
@@ -1005,16 +986,20 @@ int main(int argc, char** argv) {
}
}
leveldb::g_env = leveldb::Env::Default();
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db == NULL) {
leveldb::g_env->GetTestDirectory(&default_db_path);
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench";
FLAGS_db = default_db_path.c_str();
}
leveldb::Benchmark benchmark;
benchmark.Run();
// benchmark class needs to destruct before Shutdown call
{
leveldb::Benchmark benchmark;
benchmark.Run();
}
leveldb::Env::Shutdown();
return 0;
}
+1307 -459
View File
File diff suppressed because it is too large. Load diff
+83 -42
View File
@@ -13,7 +13,7 @@
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "port/port.h"
#include "port/thread_annotations.h"
#include "util/cache2.h"
namespace leveldb {
@@ -29,26 +29,37 @@ class DBImpl : public DB {
virtual ~DBImpl();
// Implementations of the DB interface
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value, const KeyMetaData * meta=NULL);
virtual Status Delete(const WriteOptions&, const Slice& key);
virtual Status Write(const WriteOptions& options, WriteBatch* updates);
virtual Status Get(const ReadOptions& options,
const Slice& key,
std::string* value);
std::string* value,
KeyMetaData * meta=NULL);
virtual Status Get(const ReadOptions& options,
const Slice& key,
Value* value,
KeyMetaData * meta=NULL);
virtual Iterator* NewIterator(const ReadOptions&);
virtual const Snapshot* GetSnapshot();
virtual void ReleaseSnapshot(const Snapshot* snapshot);
virtual bool GetProperty(const Slice& property, std::string* value);
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
virtual void CompactRange(const Slice* begin, const Slice* end);
virtual Status VerifyLevels();
virtual void CheckAvailableCompactions();
virtual Logger* GetLogger() const { return options_.info_log; }
// Extra methods (for testing) that are not in the public DB interface
const Options & GetOptions() const { return options_; };
// Compact any files in the named level that overlap [*begin,*end]
void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
// Force current memtable contents to be compacted.
Status TEST_CompactMemTable();
// Force current memtable contents to be compacted, waits for completion
Status CompactMemTableSynchronous();
Status TEST_CompactMemTable(); // wraps CompactMemTableSynchronous (historical)
// Return an internal iterator over the current state of the database.
// The keys of this iterator are internal keys (see format.h).
@@ -59,64 +70,82 @@ class DBImpl : public DB {
// file at a level >= 1.
int64_t TEST_MaxNextLevelOverlappingBytes();
// Record a sample of bytes read at the specified internal key.
// Samples are taken approximately once every config::kReadBytesPeriod
// bytes.
void RecordReadSample(Slice key);
// These are routines that DBListImpl calls across all open databases
void ResizeCaches() {double_cache.ResizeCaches();};
size_t GetCacheCapacity() {return(double_cache.GetCapacity(false));}
void PurgeExpiredFileCache() {double_cache.PurgeExpiredFiles();};
private:
// in util/hot_backup.cc
void HotBackup();
bool PurgeWriteBuffer();
bool WriteBackupManifest();
bool CreateBackupLinks(Version * Version, Options & BackupOptions);
bool CopyLOGSegment(long FileEnd);
void HotBackupComplete();
void BackgroundCall2(Compaction * Compact);
void BackgroundImmCompactCall();
bool IsCompactionScheduled();
uint32_t RunningCompactionCount() {mutex_.AssertHeld(); return(running_compactions_);};
protected:
friend class DB;
struct CompactionState;
struct Writer;
Iterator* NewInternalIterator(const ReadOptions&,
SequenceNumber* latest_snapshot,
uint32_t* seed);
SequenceNumber* latest_snapshot);
Status NewDB();
// Recover the descriptor from persistent storage. May do a significant
// amount of work to recover recently logged updates. Any changes to
// be made to the descriptor are added to *edit.
Status Recover(VersionEdit* edit, bool* save_manifest)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status Recover(VersionEdit* edit);
// Riak routine: pause DB::Open if too many compactions
// stacked up immediately. Happens in some repairs and
// some Riak upgrades
void CheckCompactionState();
void MaybeIgnoreError(Status* s) const;
// Delete any unneeded files and stale in-memory entries.
void DeleteObsoleteFiles();
void KeepOrDelete(const std::string & Filename, int level, const std::set<uint64_t> & Live);
// Compact the in-memory write buffer to disk. Switches to a new
// log-file/memtable and writes a new descriptor iff successful.
// Errors are recorded in bg_error_.
void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status CompactMemTable();
Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest,
VersionEdit* edit, SequenceNumber* max_sequence)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status RecoverLogFile(uint64_t log_number,
VersionEdit* edit,
SequenceNumber* max_sequence);
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status WriteLevel0Table(volatile MemTable* mem, VersionEdit* edit, Version* base);
Status MakeRoomForWrite(bool force /* TRUE forces memtable rotation to disk (for testing) */);
Status NewRecoveryLog(uint64_t NewLogNumber);
Status MakeRoomForWrite(bool force /* compact even if there is room? */)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
WriteBatch* BuildBatchGroup(Writer** last_writer);
void RecordBackgroundError(const Status& s);
void MaybeScheduleCompaction();
void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
static void BGWork(void* db);
void BackgroundCall();
void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void CleanupCompaction(CompactionState* compact)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status DoCompactionWork(CompactionState* compact)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status BackgroundCompaction(Compaction * Compact=NULL);
Status BackgroundExpiry(Compaction * Compact=NULL);
Status OpenCompactionOutputFile(CompactionState* compact);
void CleanupCompaction(CompactionState* compact);
Status DoCompactionWork(CompactionState* compact);
int64_t PrioritizeWork(bool IsLevel0);
Status OpenCompactionOutputFile(CompactionState* compact, size_t sample_value_size);
bool Send2PageCache(CompactionState * compact);
size_t MaybeRaiseBlockSize(Compaction & CompactionStuff, size_t SampleValueSize);
Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
Status InstallCompactionResults(CompactionState* compact)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status InstallCompactionResults(CompactionState* compact);
// initialized before options so its block_cache is available
class DoubleCache double_cache;
// Constant after construction
Env* const env_;
@@ -130,20 +159,22 @@ class DBImpl : public DB {
// table_cache_ provides its own synchronization
TableCache* table_cache_;
// Lock over the persistent DB state. Non-NULL iff successfully acquired.
FileLock* db_lock_;
// State below is protected by mutex_
port::Mutex mutex_;
port::Mutex throttle_mutex_; // used by write throttle to force sequential waits on callers
port::AtomicPointer shutting_down_;
port::CondVar bg_cv_; // Signalled when background work finishes
MemTable* mem_;
MemTable* imm_; // Memtable being compacted
volatile MemTable* imm_; // Memtable being compacted
port::AtomicPointer has_imm_; // So bg thread can detect non-NULL imm_
WritableFile* logfile_;
uint64_t logfile_number_;
log::Writer* log_;
uint32_t seed_; // For sampling.
// Queue of writers.
std::deque<Writer*> writers_;
@@ -155,9 +186,6 @@ class DBImpl : public DB {
// part of ongoing compactions.
std::set<uint64_t> pending_outputs_;
// Has a background compaction been scheduled or is running?
bool bg_compaction_scheduled_;
// Information for a manual compaction
struct ManualCompaction {
int level;
@@ -166,7 +194,7 @@ class DBImpl : public DB {
const InternalKey* end; // NULL means end of key range
InternalKey tmp_storage; // Used to keep track of compaction progress
};
ManualCompaction* manual_compaction_;
volatile ManualCompaction* manual_compaction_;
VersionSet* versions_;
@@ -190,6 +218,18 @@ class DBImpl : public DB {
};
CompactionStats stats_[config::kNumLevels];
volatile uint64_t throttle_end;
volatile uint32_t running_compactions_;
volatile size_t current_block_size_; // last dynamic block size computed
volatile uint64_t block_size_changed_; // NowMicros() when block size computed
volatile uint64_t last_low_mem_; // NowMicros() when low memory last seen
// accessor to new, dynamic block_cache
Cache * block_cache() {return(double_cache.GetBlockCache());};
Cache * file_cache() {return(double_cache.GetFileCache());};
volatile bool hotbackup_pending_;
// No copying allowed
DBImpl(const DBImpl&);
void operator=(const DBImpl&);
@@ -204,7 +244,8 @@ class DBImpl : public DB {
extern Options SanitizeOptions(const std::string& db,
const InternalKeyComparator* icmp,
const InternalFilterPolicy* ipolicy,
const Options& src);
const Options& src,
Cache * block_cache);
} // namespace leveldb
+58 -34
View File
@@ -5,14 +5,14 @@
#include "db/db_iter.h"
#include "db/filename.h"
#include "db/db_impl.h"
#include "db/dbformat.h"
#include "leveldb/env.h"
#include "leveldb/expiry.h"
#include "leveldb/iterator.h"
#include "leveldb/perf_count.h"
#include "port/port.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/random.h"
namespace leveldb {
@@ -48,18 +48,20 @@ class DBIter: public Iterator {
kReverse
};
DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
uint32_t seed)
: db_(db),
DBIter(const std::string* dbname, Env* env,
const Comparator* cmp, Iterator* iter, SequenceNumber s,
const ExpiryModule * expiry)
: dbname_(dbname),
env_(env),
user_comparator_(cmp),
iter_(iter),
sequence_(s),
direction_(kForward),
valid_(false),
rnd_(seed),
bytes_counter_(RandomPeriod()) {
expiry_(expiry) {
}
virtual ~DBIter() {
gPerfCounters->Inc(ePerfIterDelete);
delete iter_;
}
virtual bool Valid() const { return valid_; }
@@ -71,6 +73,26 @@ class DBIter: public Iterator {
assert(valid_);
return (direction_ == kForward) ? iter_->value() : saved_value_;
}
// Riak specific: if a database iterator, returns key meta data
// REQUIRES: Valid() and forward iteration
// (reverse iteration is possible, just needs code)
virtual KeyMetaData & keymetadata() const
{
assert(valid_ && kForward==direction_);
if (kForward==direction_)
{
ParsedInternalKey parsed;
// this initialization clears a warning. ParsedInternalKey says
// it is not initializing for performance reasons ... oh well
parsed.type=kTypeValue; parsed.sequence=0; parsed.expiry=0;
ParseInternalKey(iter_->key(), &parsed);
keymetadata_.m_Type=parsed.type;
keymetadata_.m_Sequence=parsed.sequence;
keymetadata_.m_Expiry=parsed.expiry;
}
return(keymetadata_);
}
virtual Status status() const {
if (status_.ok()) {
return iter_->status();
@@ -103,12 +125,8 @@ class DBIter: public Iterator {
}
}
// Pick next gap with average value of config::kReadBytesPeriod.
ssize_t RandomPeriod() {
return rnd_.Uniform(2*config::kReadBytesPeriod);
}
DBImpl* db_;
const std::string* const dbname_;
Env* const env_;
const Comparator* const user_comparator_;
Iterator* const iter_;
SequenceNumber const sequence_;
@@ -118,9 +136,7 @@ class DBIter: public Iterator {
std::string saved_value_; // == current raw value when direction_==kReverse
Direction direction_;
bool valid_;
Random rnd_;
ssize_t bytes_counter_;
const ExpiryModule * expiry_;
// No copying allowed
DBIter(const DBIter&);
@@ -128,14 +144,7 @@ class DBIter: public Iterator {
};
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
Slice k = iter_->key();
ssize_t n = k.size() + iter_->value().size();
bytes_counter_ -= n;
while (bytes_counter_ < 0) {
bytes_counter_ += RandomPeriod();
db_->RecordReadSample(k);
}
if (!ParseInternalKey(k, ikey)) {
if (!ParseInternalKey(iter_->key(), ikey)) {
status_ = Status::Corruption("corrupted internal key in DBIter");
return false;
} else {
@@ -146,6 +155,7 @@ inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
void DBIter::Next() {
assert(valid_);
gPerfCounters->Inc(ePerfIterNext);
if (direction_ == kReverse) { // Switch directions?
direction_ = kForward;
// iter_ is pointing just before the entries for this->key(),
@@ -161,13 +171,12 @@ void DBIter::Next() {
saved_key_.clear();
return;
}
// saved_key_ already contains the key to skip past.
} else {
// Store in saved_key_ the current key so we skip it below.
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
}
FindNextUserEntry(true, &saved_key_);
// Temporarily use saved_key_ as storage for key to skip.
std::string* skip = &saved_key_;
SaveKey(ExtractUserKey(iter_->key()), skip);
FindNextUserEntry(true, skip);
}
void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
@@ -177,6 +186,9 @@ void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
do {
ParsedInternalKey ikey;
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
if (IsExpiryKey(ikey.type) && NULL!=expiry_
&& expiry_->KeyRetirementCallback(ikey))
ikey.type=kTypeDeletion;
switch (ikey.type) {
case kTypeDeletion:
// Arrange to skip all upcoming entries for this key since
@@ -184,6 +196,9 @@ void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
SaveKey(ikey.user_key, skip);
skipping = true;
break;
case kTypeValueWriteTime:
case kTypeValueExplicitExpiry:
case kTypeValue:
if (skipping &&
user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
@@ -205,6 +220,7 @@ void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
void DBIter::Prev() {
assert(valid_);
gPerfCounters->Inc(ePerfIterPrev);
if (direction_ == kForward) { // Switch directions?
// iter_ is pointing at the current entry. Scan backwards until
// the key changes so we can use the normal reverse scanning code.
@@ -242,6 +258,10 @@ void DBIter::FindPrevUserEntry() {
// We encountered a non-deleted value in entries for previous keys,
break;
}
if (IsExpiryKey(ikey.type) && NULL!=expiry_
&& expiry_->KeyRetirementCallback(ikey))
ikey.type=kTypeDeletion;
value_type = ikey.type;
if (value_type == kTypeDeletion) {
saved_key_.clear();
@@ -272,11 +292,12 @@ void DBIter::FindPrevUserEntry() {
}
void DBIter::Seek(const Slice& target) {
gPerfCounters->Inc(ePerfIterSeek);
direction_ = kForward;
ClearSavedValue();
saved_key_.clear();
AppendInternalKey(
&saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
&saved_key_, ParsedInternalKey(target, 0, sequence_, kValueTypeForSeek));
iter_->Seek(saved_key_);
if (iter_->Valid()) {
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
@@ -286,6 +307,7 @@ void DBIter::Seek(const Slice& target) {
}
void DBIter::SeekToFirst() {
gPerfCounters->Inc(ePerfIterSeekFirst);
direction_ = kForward;
ClearSavedValue();
iter_->SeekToFirst();
@@ -297,6 +319,7 @@ void DBIter::SeekToFirst() {
}
void DBIter::SeekToLast() {
gPerfCounters->Inc(ePerfIterSeekLast);
direction_ = kReverse;
ClearSavedValue();
iter_->SeekToLast();
@@ -306,12 +329,13 @@ void DBIter::SeekToLast() {
} // anonymous namespace
Iterator* NewDBIterator(
DBImpl* db,
const std::string* dbname,
Env* env,
const Comparator* user_key_comparator,
Iterator* internal_iter,
SequenceNumber sequence,
uint32_t seed) {
return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
const SequenceNumber& sequence,
const ExpiryModule * expiry) {
return new DBIter(dbname, env, user_key_comparator, internal_iter, sequence, expiry);
}
} // namespace leveldb
+5 -5
View File
@@ -7,21 +7,21 @@
#include <stdint.h>
#include "leveldb/db.h"
#include "leveldb/expiry.h"
#include "db/dbformat.h"
namespace leveldb {
class DBImpl;
// Return a new iterator that converts internal keys (yielded by
// "*internal_iter") that were live at the specified "sequence" number
// into appropriate user keys.
extern Iterator* NewDBIterator(
DBImpl* db,
const std::string* dbname,
Env* env,
const Comparator* user_key_comparator,
Iterator* internal_iter,
SequenceNumber sequence,
uint32_t seed);
const SequenceNumber& sequence,
const ExpiryModule * expiry=NULL);
} // namespace leveldb
+202 -301
View File
@@ -33,11 +33,8 @@ class AtomicCounter {
public:
AtomicCounter() : count_(0) { }
void Increment() {
IncrementBy(1);
}
void IncrementBy(int count) {
MutexLock l(&mu_);
count_ += count;
count_++;
}
int Read() {
MutexLock l(&mu_);
@@ -48,20 +45,13 @@ class AtomicCounter {
count_ = 0;
}
};
void DelayMilliseconds(int millis) {
Env::Default()->SleepForMicroseconds(millis * 1000);
}
}
// Special Env used to delay background operations
class SpecialEnv : public EnvWrapper {
public:
// sstable/log Sync() calls are blocked while this pointer is non-NULL.
port::AtomicPointer delay_data_sync_;
// sstable/log Sync() calls return an error.
port::AtomicPointer data_sync_error_;
// sstable Sync() calls are blocked while this pointer is non-NULL.
port::AtomicPointer delay_sstable_sync_;
// Simulate no-space errors while this pointer is non-NULL.
port::AtomicPointer no_space_;
@@ -69,37 +59,30 @@ class SpecialEnv : public EnvWrapper {
// Simulate non-writable file system while this pointer is non-NULL
port::AtomicPointer non_writable_;
// Force sync of manifest files to fail while this pointer is non-NULL
port::AtomicPointer manifest_sync_error_;
// Force write to manifest files to fail while this pointer is non-NULL
port::AtomicPointer manifest_write_error_;
bool count_random_reads_;
AtomicCounter random_read_counter_;
AtomicCounter sleep_counter_;
explicit SpecialEnv(Env* base) : EnvWrapper(base) {
delay_data_sync_.Release_Store(NULL);
data_sync_error_.Release_Store(NULL);
delay_sstable_sync_.Release_Store(NULL);
no_space_.Release_Store(NULL);
non_writable_.Release_Store(NULL);
count_random_reads_ = false;
manifest_sync_error_.Release_Store(NULL);
manifest_write_error_.Release_Store(NULL);
}
Status NewWritableFile(const std::string& f, WritableFile** r) {
class DataFile : public WritableFile {
Status NewWritableFile(const std::string& f, WritableFile** r, size_t map_size) {
class SSTableFile : public WritableFile {
private:
SpecialEnv* env_;
WritableFile* base_;
public:
DataFile(SpecialEnv* env, WritableFile* base)
SSTableFile(SpecialEnv* env, WritableFile* base)
: env_(env),
base_(base) {
}
~DataFile() { delete base_; }
~SSTableFile() { delete base_; }
Status Append(const Slice& data) {
if (env_->no_space_.Acquire_Load() != NULL) {
// Drop writes on the floor
@@ -111,51 +94,21 @@ class SpecialEnv : public EnvWrapper {
Status Close() { return base_->Close(); }
Status Flush() { return base_->Flush(); }
Status Sync() {
if (env_->data_sync_error_.Acquire_Load() != NULL) {
return Status::IOError("simulated data sync error");
}
while (env_->delay_data_sync_.Acquire_Load() != NULL) {
DelayMilliseconds(100);
while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
env_->SleepForMicroseconds(100000);
}
return base_->Sync();
}
};
class ManifestFile : public WritableFile {
private:
SpecialEnv* env_;
WritableFile* base_;
public:
ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
~ManifestFile() { delete base_; }
Status Append(const Slice& data) {
if (env_->manifest_write_error_.Acquire_Load() != NULL) {
return Status::IOError("simulated writer error");
} else {
return base_->Append(data);
}
}
Status Close() { return base_->Close(); }
Status Flush() { return base_->Flush(); }
Status Sync() {
if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
return Status::IOError("simulated sync error");
} else {
return base_->Sync();
}
}
};
if (non_writable_.Acquire_Load() != NULL) {
return Status::IOError("simulated write error");
}
Status s = target()->NewWritableFile(f, r);
Status s = target()->NewWritableFile(f, r, 2<<20);
if (s.ok()) {
if (strstr(f.c_str(), ".ldb") != NULL ||
strstr(f.c_str(), ".log") != NULL) {
*r = new DataFile(this, *r);
} else if (strstr(f.c_str(), "MANIFEST") != NULL) {
*r = new ManifestFile(this, *r);
if (strstr(f.c_str(), ".sst") != NULL) {
*r = new SSTableFile(this, *r);
}
}
return s;
@@ -184,6 +137,11 @@ class SpecialEnv : public EnvWrapper {
}
return s;
}
virtual void SleepForMicroseconds(int micros) {
sleep_counter_.Increment();
target()->SleepForMicroseconds(micros);
}
};
class DBTest {
@@ -193,7 +151,6 @@ class DBTest {
// Sequence of option configurations to try
enum OptionConfig {
kDefault,
kReuse,
kFilter,
kUncompressed,
kEnd
@@ -209,7 +166,7 @@ class DBTest {
DBTest() : option_config_(kDefault),
env_(new SpecialEnv(Env::Default())) {
filter_policy_ = NewBloomFilterPolicy(10);
filter_policy_ = NewBloomFilterPolicy2(16);
dbname_ = test::TmpDir() + "/db_test";
DestroyDB(dbname_, Options());
db_ = NULL;
@@ -238,11 +195,7 @@ class DBTest {
// Return the current option configuration.
Options CurrentOptions() {
Options options;
options.reuse_logs = false;
switch (option_config_) {
case kReuse:
options.reuse_logs = true;
break;
case kFilter:
options.filter_policy = filter_policy_;
break;
@@ -290,6 +243,23 @@ class DBTest {
return DB::Open(opts, dbname_, &db_);
}
Status DoubleOpen(Options* options = NULL) {
DB * db_fail;
delete db_;
db_ = NULL;
Options opts, opts2;
if (options != NULL) {
opts = *options;
} else {
opts = CurrentOptions();
opts.create_if_missing = true;
}
last_options_ = opts;
DB::Open(opts, dbname_, &db_);
return DB::Open(opts2, dbname_, &db_fail);
}
Status Put(const std::string& k, const std::string& v) {
return db_->Put(WriteOptions(), k, v);
}
@@ -311,6 +281,20 @@ class DBTest {
return result;
}
std::string GetNoCache(const std::string& k, const Snapshot* snapshot = NULL) {
ReadOptions options;
options.snapshot = snapshot;
options.fill_cache=false;
std::string result;
Status s = db_->Get(options, k, &result);
if (s.IsNotFound()) {
result = "NOT_FOUND";
} else if (!s.ok()) {
result = s.ToString();
}
return result;
}
// Return a string that contains all key,value pairs in order,
// formatted like "(k1->v1)(k2->v2)".
std::string Contents() {
@@ -326,7 +310,7 @@ class DBTest {
}
// Check reverse iteration results are the reverse of forward results
size_t matched = 0;
int matched = 0;
for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
ASSERT_LT(matched, forward.size());
ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
@@ -340,7 +324,7 @@ class DBTest {
std::string AllEntriesFor(const Slice& user_key) {
Iterator* iter = dbfull()->TEST_NewInternalIterator();
InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
InternalKey target(user_key, 0, kMaxSequenceNumber, kTypeValue);
iter->Seek(target.Encode());
std::string result;
if (!iter->status().ok()) {
@@ -361,6 +345,8 @@ class DBTest {
}
first = false;
switch (ikey.type) {
case kTypeValueWriteTime:
case kTypeValueExplicitExpiry:
case kTypeValue:
result += iter->value().ToString();
break;
@@ -474,38 +460,6 @@ class DBTest {
}
return result;
}
bool DeleteAnSSTFile() {
std::vector<std::string> filenames;
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
uint64_t number;
FileType type;
for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number)));
return true;
}
}
return false;
}
// Returns number of files renamed.
int RenameLDBToSST() {
std::vector<std::string> filenames;
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
uint64_t number;
FileType type;
int files_renamed = 0;
for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
const std::string from = TableFileName(dbname_, number);
const std::string to = SSTTableFileName(dbname_, number);
ASSERT_OK(env_->RenameFile(from, to));
files_renamed++;
}
}
return files_renamed;
}
};
TEST(DBTest, Empty) {
@@ -515,6 +469,11 @@ TEST(DBTest, Empty) {
} while (ChangeOptions());
}
TEST(DBTest, DoubleOpen)
{
ASSERT_NOTOK(DoubleOpen());
}
TEST(DBTest, ReadWrite) {
do {
ASSERT_OK(Put("foo", "v1"));
@@ -547,11 +506,11 @@ TEST(DBTest, GetFromImmutableLayer) {
ASSERT_OK(Put("foo", "v1"));
ASSERT_EQ("v1", Get("foo"));
env_->delay_data_sync_.Release_Store(env_); // Block sync calls
env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
Put("k1", std::string(100000, 'x')); // Fill memtable
Put("k2", std::string(100000, 'y')); // Trigger compaction
ASSERT_EQ("v1", Get("foo"));
env_->delay_data_sync_.Release_Store(NULL); // Release sync calls
env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
} while (ChangeOptions());
}
@@ -563,17 +522,6 @@ TEST(DBTest, GetFromVersions) {
} while (ChangeOptions());
}
TEST(DBTest, GetMemUsage) {
do {
ASSERT_OK(Put("foo", "v1"));
std::string val;
ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val));
int mem_usage = atoi(val.c_str());
ASSERT_GT(mem_usage, 0);
ASSERT_LT(mem_usage, 5*1024*1024);
} while (ChangeOptions());
}
TEST(DBTest, GetSnapshot) {
do {
// Try with both a short key and a long key
@@ -634,6 +582,9 @@ TEST(DBTest, GetPicksCorrectFile) {
} while (ChangeOptions());
}
#if 0
// riak does not execute compaction due to reads
TEST(DBTest, GetEncountersEmptyLevel) {
do {
// Arrange for the following to happen:
@@ -642,7 +593,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
// * sstable B in level 2
// Then do enough Get() calls to arrange for an automatic compaction
// of sstable A. A bug would cause the compaction to be marked as
// occurring at level 1 (instead of the correct level 0).
// occuring at level 1 (instead of the correct level 0).
// Step 1: First place sstables in levels 0 and 2
int compaction_count = 0;
@@ -667,11 +618,12 @@ TEST(DBTest, GetEncountersEmptyLevel) {
}
// Step 4: Wait for compaction to finish
DelayMilliseconds(1000);
env_->SleepForMicroseconds(1000000);
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
} while (ChangeOptions());
}
#endif
TEST(DBTest, IterEmpty) {
Iterator* iter = db_->NewIterator(ReadOptions());
@@ -996,7 +948,8 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) {
dbfull()->TEST_CompactRange(0, NULL, NULL);
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
ASSERT_GT(NumTableFilesAtLevel(1), 1);
// not riak ASSERT_GT(NumTableFilesAtLevel(1), 1);
ASSERT_EQ(NumTableFilesAtLevel(1), 1); // yes riak
for (int i = 0; i < 80; i++) {
ASSERT_EQ(Get(Key(i)), values[i]);
}
@@ -1010,7 +963,8 @@ TEST(DBTest, RepeatedWritesToSameKey) {
// We must have at most one file per level except for level-0,
// which may have up to kL0_StopWritesTrigger files.
const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
// ... basho adds *2 since level-1 is now overlapped too
const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger*2;
Random rnd(301);
std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
@@ -1054,11 +1008,13 @@ TEST(DBTest, SparseMerge) {
// Compactions should not cause us to create a situation where
// a file overlaps too much data at the next level.
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
// 07/10/14 matthewv - we overlap first two levels. sparse test not appropriate there,
// and we set overlaps into 100s of megabytes as "normal"
// ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
dbfull()->TEST_CompactRange(0, NULL, NULL);
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
// ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
dbfull()->TEST_CompactRange(1, NULL, NULL);
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
// ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
}
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
@@ -1096,14 +1052,6 @@ TEST(DBTest, ApproximateSizes) {
// 0 because GetApproximateSizes() does not account for memtable space
ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
if (options.reuse_logs) {
// Recovery will reuse memtable, and GetApproximateSizes() does not
// account for memtable usage;
Reopen(&options);
ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
continue;
}
// Check sizes across recovery by reopening a few times
for (int run = 0; run < 3; run++) {
Reopen(&options);
@@ -1147,11 +1095,6 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
if (options.reuse_logs) {
// Need to force a memtable compaction since recovery does not do so.
ASSERT_OK(dbfull()->TEST_CompactMemTable());
}
// Check sizes across recovery by reopening a few times
for (int run = 0; run < 3; run++) {
Reopen(&options);
@@ -1223,7 +1166,7 @@ TEST(DBTest, Snapshot) {
ASSERT_EQ("v4", Get("foo"));
} while (ChangeOptions());
}
#if 0 // trouble under Riak due to assumed file sizes
TEST(DBTest, HiddenValuesAreRemoved) {
do {
Random rnd(301);
@@ -1254,7 +1197,7 @@ TEST(DBTest, HiddenValuesAreRemoved) {
ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
} while (ChangeOptions());
}
#endif
TEST(DBTest, DeletionMarkers1) {
Put("foo", "v1");
ASSERT_OK(dbfull()->TEST_CompactMemTable());
@@ -1271,13 +1214,14 @@ TEST(DBTest, DeletionMarkers1) {
Delete("foo");
Put("foo", "v2");
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // stays at level 0
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); // riak 1.3, DEL merged out by BuildTable
Slice z("z");
dbfull()->TEST_CompactRange(last-2, NULL, &z);
dbfull()->TEST_CompactRange(0, NULL, &z);
dbfull()->TEST_CompactRange(1, NULL, &z);
// DEL eliminated, but v1 remains because we aren't compacting that level
// (DEL can be eliminated because v2 hides v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); // Riak 1.4 has merged to level 1
dbfull()->TEST_CompactRange(last-1, NULL, NULL);
// Merging last-1 w/ last, so we are the base level for "foo", so
// DEL is removed. (as is v1).
@@ -1289,39 +1233,47 @@ TEST(DBTest, DeletionMarkers2) {
ASSERT_OK(dbfull()->TEST_CompactMemTable());
const int last = config::kMaxMemCompactLevel;
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
dbfull()->TEST_CompactRange(0, NULL, NULL);
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
ASSERT_EQ(NumTableFilesAtLevel(last-1), 0);
// Place a table at level last-1 to prevent merging with preceding mutation
Put("a", "begin");
Put("z", "end");
dbfull()->TEST_CompactMemTable();
ASSERT_EQ(NumTableFilesAtLevel(last), 1);
dbfull()->TEST_CompactMemTable(); // goes to last-1
ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
Delete("foo");
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level 0
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
dbfull()->TEST_CompactRange(last-2, NULL, NULL);
dbfull()->TEST_CompactRange(0, NULL, NULL); // Riak overlaps level 1
// DEL kept: "last" file overlaps
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
dbfull()->TEST_CompactRange(last-1, NULL, NULL);
// Merging last-1 w/ last, so we are the base level for "foo", so
// DEL is removed. (as is v1).
dbfull()->TEST_CompactRange(1, NULL, NULL);
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
dbfull()->TEST_CompactRange(2, NULL, NULL);
ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
}
TEST(DBTest, OverlapInLevel0) {
do {
ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
ASSERT_EQ(config::kMaxMemCompactLevel, 3) << "Fix test to match config";
// Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
ASSERT_OK(Put("100", "v100"));
ASSERT_OK(Put("999", "v999"));
dbfull()->TEST_CompactMemTable();
dbfull()->TEST_CompactRange(0, NULL, NULL);
dbfull()->TEST_CompactRange(1, NULL, NULL);
ASSERT_OK(Delete("100"));
ASSERT_OK(Delete("999"));
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("0,1,1", FilesPerLevel());
dbfull()->TEST_CompactRange(0, NULL, NULL);
ASSERT_EQ("0,0,1,1", FilesPerLevel());
// Make files spanning the following ranges in level-0:
// files[0] 200 .. 900
@@ -1334,7 +1286,7 @@ TEST(DBTest, OverlapInLevel0) {
ASSERT_OK(Put("600", "v600"));
ASSERT_OK(Put("900", "v900"));
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("2,1,1", FilesPerLevel());
ASSERT_EQ("2,0,1,1", FilesPerLevel());
// Compact away the placeholder files we created initially
dbfull()->TEST_CompactRange(1, NULL, NULL);
@@ -1364,7 +1316,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_a) {
Reopen();
Reopen();
ASSERT_EQ("(a->v)", Contents());
DelayMilliseconds(1000); // Wait for compaction to finish
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
ASSERT_EQ("(a->v)", Contents());
}
@@ -1380,7 +1332,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
Put("","");
Reopen();
Put("","");
DelayMilliseconds(1000); // Wait for compaction to finish
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
Reopen();
Put("d","dv");
Reopen();
@@ -1390,7 +1342,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
Delete("b");
Reopen();
ASSERT_EQ("(->)(c->cv)", Contents());
DelayMilliseconds(1000); // Wait for compaction to finish
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
ASSERT_EQ("(->)(c->cv)", Contents());
}
@@ -1473,37 +1425,37 @@ TEST(DBTest, CustomComparator) {
}
TEST(DBTest, ManualCompaction) {
ASSERT_EQ(config::kMaxMemCompactLevel, 2)
ASSERT_EQ(config::kMaxMemCompactLevel, 3)
<< "Need to update this test to match kMaxMemCompactLevel";
MakeTables(3, "p", "q");
ASSERT_EQ("1,1,1", FilesPerLevel());
ASSERT_EQ("1,0,1,1", FilesPerLevel());
// Compaction range falls before files
Compact("", "c");
ASSERT_EQ("1,1,1", FilesPerLevel());
ASSERT_EQ("0,1,1,1", FilesPerLevel());
// Compaction range falls after files
Compact("r", "z");
ASSERT_EQ("1,1,1", FilesPerLevel());
ASSERT_EQ("0,1,1,1", FilesPerLevel());
// Compaction range overlaps files
Compact("p1", "p9");
ASSERT_EQ("0,0,1", FilesPerLevel());
ASSERT_EQ("0,0,0,1", FilesPerLevel());
// Populate a different range
MakeTables(3, "c", "e");
ASSERT_EQ("1,1,2", FilesPerLevel());
ASSERT_EQ("1,0,1,2", FilesPerLevel());
// Compact just the new range
Compact("b", "f");
ASSERT_EQ("0,0,2", FilesPerLevel());
ASSERT_EQ("0,0,0,2", FilesPerLevel());
// Compact all
MakeTables(1, "a", "z");
ASSERT_EQ("0,1,2", FilesPerLevel());
ASSERT_EQ("0,0,1,2", FilesPerLevel());
db_->CompactRange(NULL, NULL);
ASSERT_EQ("0,0,1", FilesPerLevel());
ASSERT_EQ("0,0,0,1", FilesPerLevel());
}
TEST(DBTest, DBOpen_Options) {
@@ -1545,12 +1497,6 @@ TEST(DBTest, DBOpen_Options) {
db = NULL;
}
TEST(DBTest, Locking) {
DB* db2 = NULL;
Status s = DB::Open(CurrentOptions(), dbname_, &db2);
ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
}
// Check that number of files does not grow when we are out of space
TEST(DBTest, NoSpace) {
Options options = CurrentOptions();
@@ -1562,15 +1508,19 @@ TEST(DBTest, NoSpace) {
Compact("a", "z");
const int num_files = CountFiles();
env_->no_space_.Release_Store(env_); // Force out-of-space errors
for (int i = 0; i < 10; i++) {
env_->sleep_counter_.Reset();
for (int i = 0; i < 5; i++) {
for (int level = 0; level < config::kNumLevels-1; level++) {
dbfull()->TEST_CompactRange(level, NULL, NULL);
}
}
env_->no_space_.Release_Store(NULL);
ASSERT_LT(CountFiles(), num_files + 3);
}
// Check that compaction attempts slept after errors
ASSERT_GE(env_->sleep_counter_.Read(), 5);
}
#if 0
TEST(DBTest, NonWritableFileSystem) {
Options options = CurrentOptions();
options.write_buffer_size = 1000;
@@ -1584,119 +1534,13 @@ TEST(DBTest, NonWritableFileSystem) {
fprintf(stderr, "iter %d; errors %d\n", i, errors);
if (!Put("foo", big).ok()) {
errors++;
DelayMilliseconds(100);
env_->SleepForMicroseconds(100000);
}
}
ASSERT_GT(errors, 0);
env_->non_writable_.Release_Store(NULL);
}
TEST(DBTest, WriteSyncError) {
// Check that log sync errors cause the DB to disallow future writes.
// (a) Cause log sync calls to fail
Options options = CurrentOptions();
options.env = env_;
Reopen(&options);
env_->data_sync_error_.Release_Store(env_);
// (b) Normal write should succeed
WriteOptions w;
ASSERT_OK(db_->Put(w, "k1", "v1"));
ASSERT_EQ("v1", Get("k1"));
// (c) Do a sync write; should fail
w.sync = true;
ASSERT_TRUE(!db_->Put(w, "k2", "v2").ok());
ASSERT_EQ("v1", Get("k1"));
ASSERT_EQ("NOT_FOUND", Get("k2"));
// (d) make sync behave normally
env_->data_sync_error_.Release_Store(NULL);
// (e) Do a non-sync write; should fail
w.sync = false;
ASSERT_TRUE(!db_->Put(w, "k3", "v3").ok());
ASSERT_EQ("v1", Get("k1"));
ASSERT_EQ("NOT_FOUND", Get("k2"));
ASSERT_EQ("NOT_FOUND", Get("k3"));
}
TEST(DBTest, ManifestWriteError) {
// Test for the following problem:
// (a) Compaction produces file F
// (b) Log record containing F is written to MANIFEST file, but Sync() fails
// (c) GC deletes F
// (d) After reopening DB, reads fail since deleted F is named in log record
// We iterate twice. In the second iteration, everything is the
// same except the log record never makes it to the MANIFEST file.
for (int iter = 0; iter < 2; iter++) {
port::AtomicPointer* error_type = (iter == 0)
? &env_->manifest_sync_error_
: &env_->manifest_write_error_;
// Insert foo=>bar mapping
Options options = CurrentOptions();
options.env = env_;
options.create_if_missing = true;
options.error_if_exists = false;
DestroyAndReopen(&options);
ASSERT_OK(Put("foo", "bar"));
ASSERT_EQ("bar", Get("foo"));
// Memtable compaction (will succeed)
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("bar", Get("foo"));
const int last = config::kMaxMemCompactLevel;
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
// Merging compaction (will fail)
error_type->Release_Store(env_);
dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail
ASSERT_EQ("bar", Get("foo"));
// Recovery: should not lose data
error_type->Release_Store(NULL);
Reopen(&options);
ASSERT_EQ("bar", Get("foo"));
}
}
TEST(DBTest, MissingSSTFile) {
ASSERT_OK(Put("foo", "bar"));
ASSERT_EQ("bar", Get("foo"));
// Dump the memtable to disk.
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("bar", Get("foo"));
Close();
ASSERT_TRUE(DeleteAnSSTFile());
Options options = CurrentOptions();
options.paranoid_checks = true;
Status s = TryReopen(&options);
ASSERT_TRUE(!s.ok());
ASSERT_TRUE(s.ToString().find("issing") != std::string::npos)
<< s.ToString();
}
TEST(DBTest, StillReadSST) {
ASSERT_OK(Put("foo", "bar"));
ASSERT_EQ("bar", Get("foo"));
// Dump the memtable to disk.
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("bar", Get("foo"));
Close();
ASSERT_GT(RenameLDBToSST(), 0);
Options options = CurrentOptions();
options.paranoid_checks = true;
Status s = TryReopen(&options);
ASSERT_TRUE(s.ok());
ASSERT_EQ("bar", Get("foo"));
}
#endif
TEST(DBTest, FilesDeletedAfterCompaction) {
ASSERT_OK(Put("foo", "v2"));
Compact("a", "z");
@@ -1713,7 +1557,7 @@ TEST(DBTest, BloomFilter) {
Options options = CurrentOptions();
options.env = env_;
options.block_cache = NewLRUCache(0); // Prevent cache hits
options.filter_policy = NewBloomFilterPolicy(10);
options.filter_policy = NewBloomFilterPolicy2(16);
Reopen(&options);
// Populate multiple layers
@@ -1728,12 +1572,12 @@ TEST(DBTest, BloomFilter) {
dbfull()->TEST_CompactMemTable();
// Prevent auto compactions triggered by seeks
env_->delay_data_sync_.Release_Store(env_);
env_->delay_sstable_sync_.Release_Store(env_);
// Lookup present keys. Should rarely read from small sstable.
env_->random_read_counter_.Reset();
for (int i = 0; i < N; i++) {
ASSERT_EQ(Key(i), Get(Key(i)));
ASSERT_EQ(Key(i), GetNoCache(Key(i)));
}
int reads = env_->random_read_counter_.Read();
fprintf(stderr, "%d present => %d reads\n", N, reads);
@@ -1743,13 +1587,13 @@ TEST(DBTest, BloomFilter) {
// Lookup present keys. Should rarely read from either sstable.
env_->random_read_counter_.Reset();
for (int i = 0; i < N; i++) {
ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing"));
ASSERT_EQ("NOT_FOUND", GetNoCache(Key(i) + ".missing"));
}
reads = env_->random_read_counter_.Read();
fprintf(stderr, "%d missing => %d reads\n", N, reads);
ASSERT_LE(reads, 3*N/100);
env_->delay_data_sync_.Release_Store(NULL);
env_->delay_sstable_sync_.Release_Store(NULL);
Close();
delete options.block_cache;
delete options.filter_policy;
@@ -1809,7 +1653,7 @@ static void MTThreadBody(void* arg) {
ASSERT_EQ(k, key);
ASSERT_GE(w, 0);
ASSERT_LT(w, kNumThreads);
ASSERT_LE(static_cast<uintptr_t>(c), reinterpret_cast<uintptr_t>(
ASSERT_LE(c, reinterpret_cast<uintptr_t>(
t->state->counter[w].Acquire_Load()));
}
}
@@ -1834,27 +1678,35 @@ TEST(DBTest, MultiThreaded) {
// Start threads
MTThread thread[kNumThreads];
pthread_t tid;
for (int id = 0; id < kNumThreads; id++) {
thread[id].state = &mt;
thread[id].id = id;
env_->StartThread(MTThreadBody, &thread[id]);
tid=env_->StartThread(MTThreadBody, &thread[id]);
pthread_detach(tid);
}
// Let them run for a while
DelayMilliseconds(kTestSeconds * 1000);
env_->SleepForMicroseconds(kTestSeconds * 1000000);
// Stop the threads and wait for them to finish
mt.stop.Release_Store(&mt);
for (int id = 0; id < kNumThreads; id++) {
while (mt.thread_done[id].Acquire_Load() == NULL) {
DelayMilliseconds(100);
env_->SleepForMicroseconds(100000);
}
}
} while (ChangeOptions());
}
namespace {
typedef std::map<std::string, std::string> KVMap;
struct KVEntry
{
std::string m_Value;
KeyMetaData m_Meta;
};
typedef std::map<std::string, KVEntry> KVMap;
}
class ModelDB: public DB {
@@ -1866,14 +1718,21 @@ class ModelDB: public DB {
explicit ModelDB(const Options& options): options_(options) { }
~ModelDB() { }
virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
return DB::Put(o, k, v);
virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v, const KeyMetaData * meta=NULL) {
return DB::Put(o, k, v, meta);
}
virtual Status Delete(const WriteOptions& o, const Slice& key) {
return DB::Delete(o, key);
}
virtual Status Get(const ReadOptions& options,
const Slice& key, std::string* value) {
const Slice& key, std::string* value,
KeyMetaData * meta = NULL) {
assert(false); // Not implemented
return Status::NotFound(key);
}
virtual Status Get(const ReadOptions& options,
const Slice& key, Value* value,
KeyMetaData * meta = NULL) {
assert(false); // Not implemented
return Status::NotFound(key);
}
@@ -1901,8 +1760,13 @@ class ModelDB: public DB {
class Handler : public WriteBatch::Handler {
public:
KVMap* map_;
virtual void Put(const Slice& key, const Slice& value) {
(*map_)[key.ToString()] = value.ToString();
virtual void Put(const Slice& key, const Slice& value,
const ValueType & type, const ExpiryTimeMicros & expiry) {
KVEntry ent;
ent.m_Value=value.ToString();
ent.m_Meta.m_Type=type;
ent.m_Meta.m_Expiry=expiry;
(*map_)[key.ToString()] = ent;
}
virtual void Delete(const Slice& key) {
map_->erase(key.ToString());
@@ -1948,7 +1812,7 @@ class ModelDB: public DB {
virtual void Next() { ++iter_; }
virtual void Prev() { --iter_; }
virtual Slice key() const { return iter_->first; }
virtual Slice value() const { return iter_->second; }
virtual Slice value() const { return iter_->second.m_Value; }
virtual Status status() const { return Status::OK(); }
private:
const KVMap* const map_;
@@ -2085,6 +1949,44 @@ TEST(DBTest, Randomized) {
} while (ChangeOptions());
}
class SimpleBugs
{
// need a class for the test harness
};
TEST(SimpleBugs, TieredRecoveryLog)
{
// DB::Open created first recovery log directly
// which lead to it NOT being in tiered storage location.
// nope std::string dbname = test::TmpDir() + "/leveldb_nontiered";
std::string dbname = "leveldb";
std::string fastname = test::TmpDir() + "/leveldb_fast";
std::string slowname = test::TmpDir() + "/leveldb_slow";
std::string combined;
DB* db = NULL;
Options opts;
opts.tiered_slow_level = 4;
opts.tiered_fast_prefix = fastname;
opts.tiered_slow_prefix = slowname;
opts.create_if_missing = true;
Env::Default()->CreateDir(fastname);
Env::Default()->CreateDir(slowname);
Status s = DB::Open(opts, dbname, &db);
ASSERT_OK(s);
ASSERT_TRUE(db != NULL);
delete db;
DestroyDB(dbname, opts);
} // TieredRecoveryLog
std::string MakeKey(unsigned int num) {
char buf[30];
snprintf(buf, sizeof(buf), "%016u", num);
@@ -2113,14 +2015,13 @@ void BM_LogAndApply(int iters, int num_base_files) {
InternalKeyComparator cmp(BytewiseComparator());
Options options;
VersionSet vset(dbname, &options, NULL, &cmp);
bool save_manifest;
ASSERT_OK(vset.Recover(&save_manifest));
ASSERT_OK(vset.Recover());
VersionEdit vbase;
uint64_t fnum = 1;
for (int i = 0; i < num_base_files; i++) {
InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
InternalKey start(MakeKey(2*fnum), 0, 1, kTypeValue);
InternalKey limit(MakeKey(2*fnum+1), 0, 1, kTypeDeletion);
vbase.AddFile2(2, fnum++, 1 /* file size */, start, limit, 0,0,0);
}
ASSERT_OK(vset.LogAndApply(&vbase, &mu));
@@ -2129,9 +2030,9 @@ void BM_LogAndApply(int iters, int num_base_files) {
for (int i = 0; i < iters; i++) {
VersionEdit vedit;
vedit.DeleteFile(2, fnum);
InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
InternalKey start(MakeKey(2*fnum), 0, 1, kTypeValue);
InternalKey limit(MakeKey(2*fnum+1), 0, 1, kTypeDeletion);
vedit.AddFile2(2, fnum++, 1 /* file size */, start, limit, 0,0,0);
vset.LogAndApply(&vedit, &mu);
}
uint64_t stop_micros = env->NowMicros();
+158 -8
View File
@@ -3,7 +3,9 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h>
//#include "leveldb/expiry.h"
#include "db/dbformat.h"
#include "db/version_set.h"
#include "port/port.h"
#include "util/coding.h"
@@ -11,26 +13,66 @@ namespace leveldb {
static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
assert(seq <= kMaxSequenceNumber);
assert(t <= kValueTypeForSeek);
// assert(t <= kValueTypeForSeek); requires revisit once expiry live
assert(t <= kTypeValueExplicitExpiry); // temp replacement for above
return (seq << 8) | t;
}
void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
result->append(key.user_key.data(), key.user_key.size());
if (IsExpiryKey(key.type))
PutFixed64(result, key.expiry);
PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
}
std::string ParsedInternalKey::DebugString() const {
char buf[50];
snprintf(buf, sizeof(buf), "' @ %llu : %d",
(unsigned long long) sequence,
int(type));
if (IsExpiryKey(type))
snprintf(buf, sizeof(buf), "' @ %llu %llu : %d",
(unsigned long long) expiry,
(unsigned long long) sequence,
int(type));
else
snprintf(buf, sizeof(buf), "' @ %llu : %d",
(unsigned long long) sequence,
int(type));
std::string result = "'";
result += EscapeString(user_key.ToString());
result += HexString(user_key.ToString());
result += buf;
return result;
}
std::string ParsedInternalKey::DebugStringHex() const {
char buf[50];
if (IsExpiryKey(type))
snprintf(buf, sizeof(buf), "' @ %llu %llu : %d",
(unsigned long long) expiry,
(unsigned long long) sequence,
int(type));
else
snprintf(buf, sizeof(buf), "' @ %llu : %d",
(unsigned long long) sequence,
int(type));
std::string result = "'";
result += HexString(user_key);
result += buf;
return result;
}
const char * KeyTypeString(ValueType val_type) {
const char * ret_ptr;
switch(val_type)
{
case kTypeDeletion: ret_ptr="kTypeDelete"; break;
case kTypeValue: ret_ptr="kTypeValue"; break;
case kTypeValueWriteTime: ret_ptr="kTypeValueWriteTime"; break;
case kTypeValueExplicitExpiry: ret_ptr="kTypeValueExplicitExpiry"; break;
default: ret_ptr="(unknown ValueType)"; break;
} // switch
return(ret_ptr);
}
std::string InternalKey::DebugString() const {
std::string result;
ParsedInternalKey parsed;
@@ -54,8 +96,10 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
// decreasing type (though sequence# should be enough to disambiguate)
int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
if (r == 0) {
const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
if (IsExpiryKey((ValueType)*(unsigned char *)&anum)) *(unsigned char*)&anum=(unsigned char)kTypeValue;
if (IsExpiryKey((ValueType)*(unsigned char *)&bnum)) *(unsigned char*)&bnum=(unsigned char)kTypeValue;
if (anum > bnum) {
r = -1;
} else if (anum < bnum) {
@@ -118,7 +162,8 @@ bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const {
return user_policy_->KeyMayMatch(ExtractUserKey(key), f);
}
LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
LookupKey::LookupKey(const Slice& user_key, SequenceNumber s, KeyMetaData * meta) {
meta_=meta;
size_t usize = user_key.size();
size_t needed = usize + 13; // A conservative estimate
char* dst;
@@ -137,4 +182,109 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
end_ = dst;
}
KeyRetirement::KeyRetirement(
const Comparator * Comparator,
SequenceNumber SmallestSnapshot,
const Options * Opts,
Compaction * const Compaction)
: has_current_user_key(false), last_sequence_for_key(kMaxSequenceNumber),
user_comparator(Comparator), smallest_snapshot(SmallestSnapshot),
options(Opts), compaction(Compaction),
valid(false), dropped(0), expired(0)
{
// NULL is ok for compaction
valid=(NULL!=user_comparator);
return;
} // KeyRetirement::KeyRetirement
KeyRetirement::~KeyRetirement()
{
if (0!=expired)
gPerfCounters->Add(ePerfExpiredKeys, expired);
} // KeyRetirement::~KeyRetirement
bool
KeyRetirement::operator()(
Slice & key)
{
ParsedInternalKey ikey;
bool drop = false, expire_flag;
if (valid)
{
if (!ParseInternalKey(key, &ikey))
{
// Do not hide error keys
current_user_key.clear();
has_current_user_key = false;
last_sequence_for_key = kMaxSequenceNumber;
} // else
else
{
if (!has_current_user_key ||
user_comparator->Compare(ikey.user_key,
Slice(current_user_key)) != 0)
{
// First occurrence of this user key
current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
has_current_user_key = true;
last_sequence_for_key = kMaxSequenceNumber;
} // if
if (last_sequence_for_key <= smallest_snapshot)
{
// Hidden by an newer entry for same user key
drop = true; // (A)
} // if
else
{
expire_flag=false;
if (NULL!=options && options->ExpiryActivated())
expire_flag=options->expiry_module->KeyRetirementCallback(ikey);
if ((ikey.type == kTypeDeletion || expire_flag)
&& ikey.sequence <= smallest_snapshot
&& NULL!=compaction // mem to level0 ignores this test
&& compaction->IsBaseLevelForKey(ikey.user_key))
{
// For this user key:
// (1) there is no data in higher levels
// (2) data in lower levels will have larger sequence numbers
// (3) data in layers that are being compacted here and have
// smaller sequence numbers will be dropped in the next
// few iterations of this loop (by rule (A) above).
// Therefore this deletion marker is obsolete and can be dropped.
drop = true;
if (expire_flag)
++expired;
else
++dropped;
} // if
} // else
last_sequence_for_key = ikey.sequence;
} // else
} // if
#if 0
// needs clean up to be used again
Log(options_.info_log,
" Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, "
"%d smallest_snapshot: %d",
ikey.user_key.ToString().c_str(),
(int)ikey.sequence, ikey.type, kTypeValue, drop,
compact->compaction->IsBaseLevelForKey(ikey.user_key),
(int)last_sequence_for_key, (int)compact->smallest_snapshot);
#endif
return(drop);
} // KeyRetirement::operator(Slice & )
} // namespace leveldb
+184 -40
View File
@@ -2,13 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
#define STORAGE_LEVELDB_DB_DBFORMAT_H_
#ifndef STORAGE_LEVELDB_DB_FORMAT_H_
#define STORAGE_LEVELDB_DB_FORMAT_H_
#include <stdio.h>
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/filter_policy.h"
#include "leveldb/options.h"
#include "leveldb/slice.h"
#include "leveldb/table_builder.h"
#include "util/coding.h"
@@ -16,19 +17,33 @@
namespace leveldb {
class Compaction;
// Grouping of constants. We may want to make some of these
// parameters set via options.
namespace config {
static const int kNumLevels = 7;
static const int kNumOverlapLevels = 2;
// Level-0 compaction is started when we hit this many files.
static const int kL0_CompactionTrigger = 4;
// Google: static const size_t kL0_CompactionTrigger = 4;
static const size_t kL0_CompactionTrigger = 6;
// Level-0 (any overlapped level) number of files where a grooming
// compaction could start
static const size_t kL0_GroomingTrigger = 4;
static const size_t kL0_GroomingTrigger10min = 2;
static const size_t kL0_GroomingTrigger20min = 1;
// ... time limits in microseconds
static const size_t kL0_Grooming10minMicros = 10 * 60 * 1000000;
static const size_t kL0_Grooming20minMicros = 20 * 60 * 1000000;
// Soft limit on number of level-0 files. We slow down writes at this point.
static const int kL0_SlowdownWritesTrigger = 8;
static const size_t kL0_SlowdownWritesTrigger = 8;
// Maximum number of level-0 files. We stop writes at this point.
static const int kL0_StopWritesTrigger = 12;
static const size_t kL0_StopWritesTrigger = 12;
// Maximum level to which a new compacted memtable is pushed if it
// does not create overlap. We try to push to level 2 to avoid the
@@ -36,31 +51,28 @@ static const int kL0_StopWritesTrigger = 12;
// expensive manifest file operations. We do not push all the way to
// the largest level since that can generate a lot of wasted disk
// space if the same key space is being repeatedly overwritten.
static const int kMaxMemCompactLevel = 2;
// Approximate gap in bytes between samples of data read during iteration.
static const int kReadBytesPeriod = 1048576;
// Basho: push to kNumOverlapLevels +1 ... beyond "landing level"
static const unsigned kMaxMemCompactLevel = kNumOverlapLevels+1;
} // namespace config
class InternalKey;
// Value types encoded as the last component of internal keys.
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
// data structures.
enum ValueType {
kTypeDeletion = 0x0,
kTypeValue = 0x1
};
// kValueTypeForSeek defines the ValueType that should be passed when
// constructing a ParsedInternalKey object for seeking to a particular
// sequence number (since we sort sequence numbers in decreasing order
// and the value type is embedded as the low 8 bits in the sequence
// number in internal keys, we need to use the highest-numbered
// ValueType, not the lowest).
// Riak note: kValueTypeForSeek is placed within temporary keys
// for comparisons. Using kTypeValueExplicitExpiry would
// force more code changes to increase internal key size.
// But ValueTypeForSeek is redundant to sequence number for
// disambiguaty. Therefore going for easiest path and NOT changing.
static const ValueType kValueTypeForSeek = kTypeValue;
typedef uint64_t SequenceNumber;
typedef uint64_t ExpiryTimeMicros;
// We leave eight bits empty at the bottom so a type and sequence#
// can be packed together into 64-bits.
@@ -69,20 +81,17 @@ static const SequenceNumber kMaxSequenceNumber =
struct ParsedInternalKey {
Slice user_key;
ExpiryTimeMicros expiry;
SequenceNumber sequence;
ValueType type;
ParsedInternalKey() { } // Intentionally left uninitialized (for speed)
ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
: user_key(u), sequence(seq), type(t) { }
ParsedInternalKey(const Slice& u, const ExpiryTimeMicros & exp, const SequenceNumber& seq, ValueType t)
: user_key(u), expiry(exp), sequence(seq), type(t) { }
std::string DebugString() const;
std::string DebugStringHex() const;
};
// Return the length of the encoding of "key".
inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
return key.user_key.size() + 8;
}
// Append the serialization of "key" to *result.
extern void AppendInternalKey(std::string* result,
const ParsedInternalKey& key);
@@ -94,20 +103,76 @@ extern void AppendInternalKey(std::string* result,
extern bool ParseInternalKey(const Slice& internal_key,
ParsedInternalKey* result);
// Returns the user key portion of an internal key.
inline Slice ExtractUserKey(const Slice& internal_key) {
assert(internal_key.size() >= 8);
return Slice(internal_key.data(), internal_key.size() - 8);
}
inline ValueType ExtractValueType(const Slice& internal_key) {
assert(internal_key.size() >= 8);
const size_t n = internal_key.size();
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
unsigned char c = num & 0xff;
unsigned char c = DecodeLeastFixed64(internal_key.data() + n - sizeof(SequenceNumber));
return static_cast<ValueType>(c);
}
inline size_t KeySuffixSize(ValueType val_type) {
size_t ret_val;
switch(val_type)
{
case kTypeDeletion:
case kTypeValue:
ret_val=sizeof(SequenceNumber);
break;
case kTypeValueWriteTime:
case kTypeValueExplicitExpiry:
ret_val=sizeof(SequenceNumber) + sizeof(ExpiryTimeMicros);
break;
default:
// assert(0); cannot use because bloom filter block's name is passed as internal key
ret_val=sizeof(SequenceNumber);
break;
} // switch
return(ret_val);
}
const char * KeyTypeString(ValueType val_type);
inline size_t KeySuffixSize(const Slice & internal_key) {
return(KeySuffixSize(ExtractValueType(internal_key)));
}
// Returns the user key portion of an internal key.
inline Slice ExtractUserKey(const Slice& internal_key) {
assert(internal_key.size() >= 8);
return Slice(internal_key.data(), internal_key.size() - KeySuffixSize(internal_key));
}
// Returns the sequence number with ValueType removed
inline SequenceNumber ExtractSequenceNumber(const Slice& internal_key) {
assert(internal_key.size() >= 8);
return(DecodeFixed64(internal_key.data() + internal_key.size() - 8)>>8);
}
// Return the length of the encoding of "key".
inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
return key.user_key.size() + KeySuffixSize(key.type);
}
// Riak: is this an expiry key and therefore contain extra ExpiryTime field
inline bool IsExpiryKey(ValueType val_type) {
return(kTypeValueWriteTime==val_type || kTypeValueExplicitExpiry==val_type);
}
// Riak: is this an expiry key and therefore contain extra ExpiryTime field
inline bool IsExpiryKey(const Slice & internal_key) {
return(internal_key.size()>=KeySuffixSize(kTypeValueWriteTime)
&& IsExpiryKey(ExtractValueType(internal_key)));
}
// Riak: extracts expiry value
inline ExpiryTimeMicros ExtractExpiry(const Slice& internal_key) {
assert(internal_key.size() >= KeySuffixSize(kTypeValueWriteTime));
assert(IsExpiryKey(internal_key));
return(DecodeFixed64(internal_key.data() + internal_key.size() - KeySuffixSize(kTypeValueWriteTime)));
}
// A comparator for internal keys that uses a specified comparator for
// the user key portion and breaks ties by decreasing sequence number.
class InternalKeyComparator : public Comparator {
@@ -129,7 +194,7 @@ class InternalKeyComparator : public Comparator {
// Filter policy wrapper that converts from internal keys to user keys
class InternalFilterPolicy : public FilterPolicy {
private:
protected:
const FilterPolicy* const user_policy_;
public:
explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { }
@@ -138,6 +203,12 @@ class InternalFilterPolicy : public FilterPolicy {
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
};
class InternalFilterPolicy2 : public InternalFilterPolicy {
public:
explicit InternalFilterPolicy2(const FilterPolicy* p) : InternalFilterPolicy(p) { }
virtual ~InternalFilterPolicy2() {delete user_policy_;};
};
// Modules in this directory should keep internal keys wrapped inside
// the following class instead of plain strings so that we do not
// incorrectly use string comparisons instead of an InternalKeyComparator.
@@ -146,8 +217,8 @@ class InternalKey {
std::string rep_;
public:
InternalKey() { } // Leave rep_ as empty to indicate it is invalid
InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
InternalKey(const Slice& user_key, ExpiryTimeMicros exp, SequenceNumber s, ValueType t) {
AppendInternalKey(&rep_, ParsedInternalKey(user_key, exp, s, t));
}
void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
@@ -157,6 +228,7 @@ class InternalKey {
}
Slice user_key() const { return ExtractUserKey(rep_); }
Slice internal_key() const { return Slice(rep_); }
void SetFrom(const ParsedInternalKey& p) {
rep_.clear();
@@ -181,8 +253,12 @@ inline bool ParseInternalKey(const Slice& internal_key,
unsigned char c = num & 0xff;
result->sequence = num >> 8;
result->type = static_cast<ValueType>(c);
result->user_key = Slice(internal_key.data(), n - 8);
return (c <= static_cast<unsigned char>(kTypeValue));
if (IsExpiryKey((ValueType)c))
result->expiry=DecodeFixed64(internal_key.data() + n - KeySuffixSize((ValueType)c));
else
result->expiry=0;
result->user_key = Slice(internal_key.data(), n - KeySuffixSize((ValueType)c));
return (c <= static_cast<unsigned char>(kTypeValueExplicitExpiry));
}
// A helper class useful for DBImpl::Get()
@@ -190,7 +266,7 @@ class LookupKey {
public:
// Initialize *this for looking up user_key at a snapshot with
// the specified sequence number.
LookupKey(const Slice& user_key, SequenceNumber sequence);
LookupKey(const Slice& user_key, SequenceNumber sequence, KeyMetaData * meta=NULL);
~LookupKey();
@@ -201,12 +277,38 @@ class LookupKey {
Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
// Return the user key
Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
Slice user_key() const
{ return Slice(kstart_, end_ - kstart_ - KeySuffixSize(internal_key())); }
// did requestor have metadata object?
bool WantsKeyMetaData() const {return(NULL!=meta_);};
void SetKeyMetaData(ValueType type, SequenceNumber seq, ExpiryTimeMicros expiry) const
{if (NULL!=meta_)
{
meta_->m_Type=type;
meta_->m_Sequence=seq;
meta_->m_Expiry=expiry;
} // if
};
void SetKeyMetaData(const ParsedInternalKey & pi_key) const
{if (NULL!=meta_)
{
meta_->m_Type=pi_key.type;
meta_->m_Sequence=pi_key.sequence;
meta_->m_Expiry=pi_key.expiry;
} // if
};
void SetKeyMetaData(const KeyMetaData & meta) const
{if (NULL!=meta_) *meta_=meta;};
private:
// We construct a char array of the form:
// klength varint32 <-- start_
// userkey char[klength] <-- kstart_
// optional uint64
// tag uint64
// <-- end_
// The array is a suitable MemTable key.
@@ -216,6 +318,9 @@ class LookupKey {
const char* end_;
char space_[200]; // Avoid allocation for short keys
// allow code that finds the key to place metadata here, even if 'const'
mutable KeyMetaData * meta_;
// No copying allowed
LookupKey(const LookupKey&);
void operator=(const LookupKey&);
@@ -223,8 +328,47 @@ class LookupKey {
inline LookupKey::~LookupKey() {
if (start_ != space_) delete[] start_;
}
};
// this class was constructed from code with DBImpl::DoCompactionWork (db_impl.cc)
// so it could be shared within BuildTable (and thus reduce Level 0 bloating)
class KeyRetirement
{
protected:
// "state" from previous key reviewed
std::string current_user_key;
bool has_current_user_key;
SequenceNumber last_sequence_for_key;
// database values needed for processing
const Comparator * user_comparator;
SequenceNumber smallest_snapshot;
const Options * options;
Compaction * const compaction;
bool valid;
size_t dropped; // tombstone or old version dropped
size_t expired; // expired dropped
public:
KeyRetirement(const Comparator * UserComparator, SequenceNumber SmallestSnapshot,
const Options * Opts, Compaction * const Compaction=NULL);
virtual ~KeyRetirement();
bool operator()(Slice & key);
size_t GetDroppedCount() const {return(dropped);};
size_t GetExpiredCount() const {return(expired);};
private:
KeyRetirement();
KeyRetirement(const KeyRetirement &);
const KeyRetirement & operator=(const KeyRetirement &);
}; // class KeyRetirement
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_
#endif // STORAGE_LEVELDB_DB_FORMAT_H_
+35 -33
View File
@@ -9,10 +9,11 @@
namespace leveldb {
static std::string IKey(const std::string& user_key,
ExpiryTimeMicros exp,
uint64_t seq,
ValueType vt) {
std::string encoded;
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
AppendInternalKey(&encoded, ParsedInternalKey(user_key, exp, seq, vt));
return encoded;
}
@@ -29,12 +30,13 @@ static std::string ShortSuccessor(const std::string& s) {
}
static void TestKey(const std::string& key,
ExpiryTimeMicros exp,
uint64_t seq,
ValueType vt) {
std::string encoded = IKey(key, seq, vt);
std::string encoded = IKey(key, exp, seq, vt);
Slice in(encoded);
ParsedInternalKey decoded("", 0, kTypeValue);
ParsedInternalKey decoded("", 0, 0, kTypeValue);
ASSERT_TRUE(ParseInternalKey(in, &decoded));
ASSERT_EQ(key, decoded.user_key.ToString());
@@ -56,53 +58,53 @@ TEST(FormatTest, InternalKey_EncodeDecode) {
};
for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
TestKey(keys[k], seq[s], kTypeValue);
TestKey("hello", 1, kTypeDeletion);
TestKey(keys[k], 0, seq[s], kTypeValue);
TestKey("hello", 0, 1, kTypeDeletion);
}
}
}
TEST(FormatTest, InternalKeyShortSeparator) {
// When user keys are same
ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue),
IKey("foo", 99, kTypeValue)));
ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue),
IKey("foo", 101, kTypeValue)));
ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue),
IKey("foo", 100, kTypeValue)));
ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue),
IKey("foo", 100, kTypeDeletion)));
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("foo", 0, 99, kTypeValue)));
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("foo", 0, 101, kTypeValue)));
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("foo", 0, 100, kTypeValue)));
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("foo", 0, 100, kTypeDeletion)));
// When user keys are misordered
ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue),
IKey("bar", 99, kTypeValue)));
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("bar", 0, 99, kTypeValue)));
// When user keys are different, but correctly ordered
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
Shorten(IKey("foo", 100, kTypeValue),
IKey("hello", 200, kTypeValue)));
ASSERT_EQ(IKey("g", 0, kMaxSequenceNumber, kValueTypeForSeek),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("hello", 0, 200, kTypeValue)));
// When start user key is prefix of limit user key
ASSERT_EQ(IKey("foo", 100, kTypeValue),
Shorten(IKey("foo", 100, kTypeValue),
IKey("foobar", 200, kTypeValue)));
ASSERT_EQ(IKey("foo", 0, 100, kTypeValue),
Shorten(IKey("foo", 0, 100, kTypeValue),
IKey("foobar", 0, 200, kTypeValue)));
// When limit user key is prefix of start user key
ASSERT_EQ(IKey("foobar", 100, kTypeValue),
Shorten(IKey("foobar", 100, kTypeValue),
IKey("foo", 200, kTypeValue)));
ASSERT_EQ(IKey("foobar", 0, 100, kTypeValue),
Shorten(IKey("foobar", 0, 100, kTypeValue),
IKey("foo", 0, 200, kTypeValue)));
}
TEST(FormatTest, InternalKeyShortestSuccessor) {
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
ShortSuccessor(IKey("foo", 100, kTypeValue)));
ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue),
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
ASSERT_EQ(IKey("g", 0, kMaxSequenceNumber, kValueTypeForSeek),
ShortSuccessor(IKey("foo", 0, 100, kTypeValue)));
ASSERT_EQ(IKey("\xff\xff", 0, 100, kTypeValue),
ShortSuccessor(IKey("\xff\xff", 0, 100, kTypeValue)));
}
} // namespace leveldb
-554
View File
@@ -1,554 +0,0 @@
// Copyright 2014 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
// This test uses a custom Env to keep track of the state of a filesystem as of
// the last "sync". It then checks for data loss errors by purposely dropping
// file data (or entire files) not protected by a "sync".
#include "leveldb/db.h"
#include <map>
#include <set>
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/log_format.h"
#include "db/version_set.h"
#include "leveldb/cache.h"
#include "leveldb/env.h"
#include "leveldb/table.h"
#include "leveldb/write_batch.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace leveldb {
static const int kValueSize = 1000;
static const int kMaxNumValues = 2000;
static const size_t kNumIterations = 3;
class FaultInjectionTestEnv;
namespace {
// Assume a filename, and not a directory name like "/foo/bar/"
static std::string GetDirName(const std::string filename) {
size_t found = filename.find_last_of("/\\");
if (found == std::string::npos) {
return "";
} else {
return filename.substr(0, found);
}
}
Status SyncDir(const std::string& dir) {
// As this is a test it isn't required to *actually* sync this directory.
return Status::OK();
}
// A basic file truncation function suitable for this test.
Status Truncate(const std::string& filename, uint64_t length) {
leveldb::Env* env = leveldb::Env::Default();
SequentialFile* orig_file;
Status s = env->NewSequentialFile(filename, &orig_file);
if (!s.ok())
return s;
char* scratch = new char[length];
leveldb::Slice result;
s = orig_file->Read(length, &result, scratch);
delete orig_file;
if (s.ok()) {
std::string tmp_name = GetDirName(filename) + "/truncate.tmp";
WritableFile* tmp_file;
s = env->NewWritableFile(tmp_name, &tmp_file);
if (s.ok()) {
s = tmp_file->Append(result);
delete tmp_file;
if (s.ok()) {
s = env->RenameFile(tmp_name, filename);
} else {
env->DeleteFile(tmp_name);
}
}
}
delete[] scratch;
return s;
}
struct FileState {
std::string filename_;
ssize_t pos_;
ssize_t pos_at_last_sync_;
ssize_t pos_at_last_flush_;
FileState(const std::string& filename)
: filename_(filename),
pos_(-1),
pos_at_last_sync_(-1),
pos_at_last_flush_(-1) { }
FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; }
Status DropUnsyncedData() const;
};
} // anonymous namespace
// A wrapper around WritableFile which informs another Env whenever this file
// is written to or sync'ed.
class TestWritableFile : public WritableFile {
public:
TestWritableFile(const FileState& state,
WritableFile* f,
FaultInjectionTestEnv* env);
virtual ~TestWritableFile();
virtual Status Append(const Slice& data);
virtual Status Close();
virtual Status Flush();
virtual Status Sync();
private:
FileState state_;
WritableFile* target_;
bool writable_file_opened_;
FaultInjectionTestEnv* env_;
Status SyncParent();
};
class FaultInjectionTestEnv : public EnvWrapper {
public:
FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {}
virtual ~FaultInjectionTestEnv() { }
virtual Status NewWritableFile(const std::string& fname,
WritableFile** result);
virtual Status NewAppendableFile(const std::string& fname,
WritableFile** result);
virtual Status DeleteFile(const std::string& f);
virtual Status RenameFile(const std::string& s, const std::string& t);
void WritableFileClosed(const FileState& state);
Status DropUnsyncedFileData();
Status DeleteFilesCreatedAfterLastDirSync();
void DirWasSynced();
bool IsFileCreatedSinceLastDirSync(const std::string& filename);
void ResetState();
void UntrackFile(const std::string& f);
// Setting the filesystem to inactive is the test equivalent to simulating a
// system reset. Setting to inactive will freeze our saved filesystem state so
// that it will stop being recorded. It can then be reset back to the state at
// the time of the reset.
bool IsFilesystemActive() const { return filesystem_active_; }
void SetFilesystemActive(bool active) { filesystem_active_ = active; }
private:
port::Mutex mutex_;
std::map<std::string, FileState> db_file_state_;
std::set<std::string> new_files_since_last_dir_sync_;
bool filesystem_active_; // Record flushes, syncs, writes
};
TestWritableFile::TestWritableFile(const FileState& state,
WritableFile* f,
FaultInjectionTestEnv* env)
: state_(state),
target_(f),
writable_file_opened_(true),
env_(env) {
assert(f != NULL);
}
TestWritableFile::~TestWritableFile() {
if (writable_file_opened_) {
Close();
}
delete target_;
}
Status TestWritableFile::Append(const Slice& data) {
Status s = target_->Append(data);
if (s.ok() && env_->IsFilesystemActive()) {
state_.pos_ += data.size();
}
return s;
}
Status TestWritableFile::Close() {
writable_file_opened_ = false;
Status s = target_->Close();
if (s.ok()) {
env_->WritableFileClosed(state_);
}
return s;
}
Status TestWritableFile::Flush() {
Status s = target_->Flush();
if (s.ok() && env_->IsFilesystemActive()) {
state_.pos_at_last_flush_ = state_.pos_;
}
return s;
}
Status TestWritableFile::SyncParent() {
Status s = SyncDir(GetDirName(state_.filename_));
if (s.ok()) {
env_->DirWasSynced();
}
return s;
}
Status TestWritableFile::Sync() {
if (!env_->IsFilesystemActive()) {
return Status::OK();
}
// Ensure new files referred to by the manifest are in the filesystem.
Status s = target_->Sync();
if (s.ok()) {
state_.pos_at_last_sync_ = state_.pos_;
}
if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) {
Status ps = SyncParent();
if (s.ok() && !ps.ok()) {
s = ps;
}
}
return s;
}
Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
WritableFile** result) {
WritableFile* actual_writable_file;
Status s = target()->NewWritableFile(fname, &actual_writable_file);
if (s.ok()) {
FileState state(fname);
state.pos_ = 0;
*result = new TestWritableFile(state, actual_writable_file, this);
// NewWritableFile doesn't append to files, so if the same file is
// opened again then it will be truncated - so forget our saved
// state.
UntrackFile(fname);
MutexLock l(&mutex_);
new_files_since_last_dir_sync_.insert(fname);
}
return s;
}
Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
WritableFile** result) {
WritableFile* actual_writable_file;
Status s = target()->NewAppendableFile(fname, &actual_writable_file);
if (s.ok()) {
FileState state(fname);
state.pos_ = 0;
{
MutexLock l(&mutex_);
if (db_file_state_.count(fname) == 0) {
new_files_since_last_dir_sync_.insert(fname);
} else {
state = db_file_state_[fname];
}
}
*result = new TestWritableFile(state, actual_writable_file, this);
}
return s;
}
Status FaultInjectionTestEnv::DropUnsyncedFileData() {
Status s;
MutexLock l(&mutex_);
for (std::map<std::string, FileState>::const_iterator it =
db_file_state_.begin();
s.ok() && it != db_file_state_.end(); ++it) {
const FileState& state = it->second;
if (!state.IsFullySynced()) {
s = state.DropUnsyncedData();
}
}
return s;
}
void FaultInjectionTestEnv::DirWasSynced() {
MutexLock l(&mutex_);
new_files_since_last_dir_sync_.clear();
}
bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync(
const std::string& filename) {
MutexLock l(&mutex_);
return new_files_since_last_dir_sync_.find(filename) !=
new_files_since_last_dir_sync_.end();
}
void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
MutexLock l(&mutex_);
db_file_state_.erase(f);
new_files_since_last_dir_sync_.erase(f);
}
Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
Status s = EnvWrapper::DeleteFile(f);
ASSERT_OK(s);
if (s.ok()) {
UntrackFile(f);
}
return s;
}
Status FaultInjectionTestEnv::RenameFile(const std::string& s,
const std::string& t) {
Status ret = EnvWrapper::RenameFile(s, t);
if (ret.ok()) {
MutexLock l(&mutex_);
if (db_file_state_.find(s) != db_file_state_.end()) {
db_file_state_[t] = db_file_state_[s];
db_file_state_.erase(s);
}
if (new_files_since_last_dir_sync_.erase(s) != 0) {
assert(new_files_since_last_dir_sync_.find(t) ==
new_files_since_last_dir_sync_.end());
new_files_since_last_dir_sync_.insert(t);
}
}
return ret;
}
void FaultInjectionTestEnv::ResetState() {
// Since we are not destroying the database, the existing files
// should keep their recorded synced/flushed state. Therefore
// we do not reset db_file_state_ and new_files_since_last_dir_sync_.
MutexLock l(&mutex_);
SetFilesystemActive(true);
}
Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
// Because DeleteFile access this container make a copy to avoid deadlock
mutex_.Lock();
std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
new_files_since_last_dir_sync_.end());
mutex_.Unlock();
Status s;
std::set<std::string>::const_iterator it;
for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) {
s = DeleteFile(*it);
}
return s;
}
void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
MutexLock l(&mutex_);
db_file_state_[state.filename_] = state;
}
Status FileState::DropUnsyncedData() const {
ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
return Truncate(filename_, sync_pos);
}
class FaultInjectionTest {
public:
enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR };
enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES };
FaultInjectionTestEnv* env_;
std::string dbname_;
Cache* tiny_cache_;
Options options_;
DB* db_;
FaultInjectionTest()
: env_(new FaultInjectionTestEnv),
tiny_cache_(NewLRUCache(100)),
db_(NULL) {
dbname_ = test::TmpDir() + "/fault_test";
DestroyDB(dbname_, Options()); // Destroy any db from earlier run
options_.reuse_logs = true;
options_.env = env_;
options_.paranoid_checks = true;
options_.block_cache = tiny_cache_;
options_.create_if_missing = true;
}
~FaultInjectionTest() {
CloseDB();
DestroyDB(dbname_, Options());
delete tiny_cache_;
delete env_;
}
void ReuseLogs(bool reuse) {
options_.reuse_logs = reuse;
}
void Build(int start_idx, int num_vals) {
std::string key_space, value_space;
WriteBatch batch;
for (int i = start_idx; i < start_idx + num_vals; i++) {
Slice key = Key(i, &key_space);
batch.Clear();
batch.Put(key, Value(i, &value_space));
WriteOptions options;
ASSERT_OK(db_->Write(options, &batch));
}
}
Status ReadValue(int i, std::string* val) const {
std::string key_space, value_space;
Slice key = Key(i, &key_space);
Value(i, &value_space);
ReadOptions options;
return db_->Get(options, key, val);
}
Status Verify(int start_idx, int num_vals,
ExpectedVerifResult expected) const {
std::string val;
std::string value_space;
Status s;
for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) {
Value(i, &value_space);
s = ReadValue(i, &val);
if (expected == VAL_EXPECT_NO_ERROR) {
if (s.ok()) {
ASSERT_EQ(value_space, val);
}
} else if (s.ok()) {
fprintf(stderr, "Expected an error at %d, but was OK\n", i);
s = Status::IOError(dbname_, "Expected value error:");
} else {
s = Status::OK(); // An expected error
}
}
return s;
}
// Return the ith key
Slice Key(int i, std::string* storage) const {
char buf[100];
snprintf(buf, sizeof(buf), "%016d", i);
storage->assign(buf, strlen(buf));
return Slice(*storage);
}
// Return the value to associate with the specified key
Slice Value(int k, std::string* storage) const {
Random r(k);
return test::RandomString(&r, kValueSize, storage);
}
Status OpenDB() {
delete db_;
db_ = NULL;
env_->ResetState();
return DB::Open(options_, dbname_, &db_);
}
void CloseDB() {
delete db_;
db_ = NULL;
}
void DeleteAllData() {
Iterator* iter = db_->NewIterator(ReadOptions());
WriteOptions options;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ASSERT_OK(db_->Delete(WriteOptions(), iter->key()));
}
delete iter;
}
void ResetDBState(ResetMethod reset_method) {
switch (reset_method) {
case RESET_DROP_UNSYNCED_DATA:
ASSERT_OK(env_->DropUnsyncedFileData());
break;
case RESET_DELETE_UNSYNCED_FILES:
ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync());
break;
default:
assert(false);
}
}
void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) {
DeleteAllData();
Build(0, num_pre_sync);
db_->CompactRange(NULL, NULL);
Build(num_pre_sync, num_post_sync);
}
void PartialCompactTestReopenWithFault(ResetMethod reset_method,
int num_pre_sync,
int num_post_sync) {
env_->SetFilesystemActive(false);
CloseDB();
ResetDBState(reset_method);
ASSERT_OK(OpenDB());
ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR));
}
void NoWriteTestPreFault() {
}
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
CloseDB();
ResetDBState(reset_method);
ASSERT_OK(OpenDB());
}
void DoTest() {
Random rnd(0);
ASSERT_OK(OpenDB());
for (size_t idx = 0; idx < kNumIterations; idx++) {
int num_pre_sync = rnd.Uniform(kMaxNumValues);
int num_post_sync = rnd.Uniform(kMaxNumValues);
PartialCompactTestPreFault(num_pre_sync, num_post_sync);
PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA,
num_pre_sync,
num_post_sync);
NoWriteTestPreFault();
NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA);
PartialCompactTestPreFault(num_pre_sync, num_post_sync);
// No new files created so we expect all values since no files will be
// dropped.
PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
num_pre_sync + num_post_sync,
0);
NoWriteTestPreFault();
NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
}
}
};
TEST(FaultInjectionTest, FaultTestNoLogReuse) {
ReuseLogs(false);
DoTest();
}
TEST(FaultInjectionTest, FaultTestWithLogReuse) {
ReuseLogs(true);
DoTest();
}
} // namespace leveldb
int main(int argc, char** argv) {
return leveldb::test::RunAllTests();
}
+174 -9
View File
@@ -4,9 +4,14 @@
#include <ctype.h>
#include <stdio.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "db/filename.h"
#include "db/dbformat.h"
#include "db/version_set.h"
#include "leveldb/env.h"
#include "leveldb/status.h"
#include "util/logging.h"
namespace leveldb {
@@ -24,19 +29,50 @@ static std::string MakeFileName(const std::string& name, uint64_t number,
return name + buf;
}
static std::string MakeFileName2(const Options & options, uint64_t number,
int level, const char* suffix) {
char buf[100];
if (0<=level)
snprintf(buf, sizeof(buf), "/%s_%-d/%06llu.%s",
suffix, level,
static_cast<unsigned long long>(number),
suffix);
else if (-1==level)
snprintf(buf, sizeof(buf), "/%s/%06llu.%s",
suffix,
static_cast<unsigned long long>(number),
suffix);
else if (-2==level)
snprintf(buf, sizeof(buf), "/%06llu.%s",
static_cast<unsigned long long>(number),
suffix);
return((level<(int)options.tiered_slow_level ?
options.tiered_fast_prefix : options.tiered_slow_prefix) + buf);
}
std::string MakeDirName2(const Options & options,
int level, const char* suffix) {
char buf[100];
if (-1!=level)
snprintf(buf, sizeof(buf), "/%s_%-d",
suffix, level);
else
snprintf(buf, sizeof(buf), "/%s",
suffix);
return((level<(int)options.tiered_slow_level ?
options.tiered_fast_prefix : options.tiered_slow_prefix) + buf);
}
std::string LogFileName(const std::string& name, uint64_t number) {
assert(number > 0);
return MakeFileName(name, number, "log");
}
std::string TableFileName(const std::string& name, uint64_t number) {
std::string TableFileName(const Options & options, uint64_t number, int level) {
assert(number > 0);
return MakeFileName(name, number, "ldb");
}
std::string SSTTableFileName(const std::string& name, uint64_t number) {
assert(number > 0);
return MakeFileName(name, number, "sst");
return MakeFileName2(options, number, level, "sst");
}
std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
@@ -69,6 +105,36 @@ std::string OldInfoLogFileName(const std::string& dbname) {
return dbname + "/LOG.old";
}
//
std::string CowFileName(const std::string& dbname) {
return dbname + "/COW";
}
// Append appropriate "backup" string to input path
std::string BackupPath(const std::string& dbname, int backup_num) {
std::string dirname;
char buf[100];
if (0 != backup_num)
snprintf(buf, sizeof(buf), "/backup.%-d", backup_num);
else
snprintf(buf, sizeof(buf), "/backup");
return(dbname + buf);
}
// update tiered_fast_prefix and tiered_slow_prefix members of
// given Options object to point to desired backup path
bool SetBackupPaths(Options & options, int backup_num) {
options.tiered_fast_prefix = BackupPath(options.tiered_fast_prefix, backup_num);
options.tiered_slow_prefix = BackupPath(options.tiered_slow_prefix, backup_num);
return(true);
}
// Owned filenames have the form:
// dbname/CURRENT
@@ -76,7 +142,8 @@ std::string OldInfoLogFileName(const std::string& dbname) {
// dbname/LOG
// dbname/LOG.old
// dbname/MANIFEST-[0-9]+
// dbname/[0-9]+.(log|sst|ldb)
// dbname/[0-9]+.(log|sst)
// dbname/COW
bool ParseFileName(const std::string& fname,
uint64_t* number,
FileType* type) {
@@ -84,6 +151,9 @@ bool ParseFileName(const std::string& fname,
if (rest == "CURRENT") {
*number = 0;
*type = kCurrentFile;
} else if (rest == "COW") {
*number = 0;
*type = kCacheWarming;
} else if (rest == "LOCK") {
*number = 0;
*type = kDBLockFile;
@@ -111,7 +181,7 @@ bool ParseFileName(const std::string& fname,
Slice suffix = rest;
if (suffix == Slice(".log")) {
*type = kLogFile;
} else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
} else if (suffix == Slice(".sst")) {
*type = kTableFile;
} else if (suffix == Slice(".dbtmp")) {
*type = kTempFile;
@@ -141,4 +211,99 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
return s;
}
Status
MakeLevelDirectories(Env * env, const Options & options)
{
Status ret_stat;
int level;
std::string dirname;
for (level=0; level<config::kNumLevels && ret_stat.ok(); ++level)
{
dirname=MakeDirName2(options, level, "sst");
// ignoring error since no way to tell if "bad" error, or "already exists" error
env->CreateDir(dirname.c_str());
} // for
return(ret_stat);
} // MakeLevelDirectories
bool
TestForLevelDirectories(
Env * env,
const Options & options,
Version * version)
{
bool ret_flag, again;
int level;
std::string dirname;
ret_flag=true;
again=true;
// walk backwards, fault will be in higher levels if partial conversion
for (level=config::kNumLevels-1; 0<=level && again; --level)
{
again=false;
// does directory exist
dirname=MakeDirName2(options, level, "sst");
ret_flag=env->FileExists(dirname.c_str());
// do all files exist in level
if (ret_flag)
{
const std::vector<FileMetaData*> & level_files(version->GetFileList(level));
std::vector<FileMetaData*>::const_iterator it;
std::string table_name;
Status s;
for (it=level_files.begin(); level_files.end()!=it && ret_flag; ++it)
{
table_name=TableFileName(options, (*it)->number, level);
ret_flag=env->FileExists(table_name.c_str());
} // for
again=ret_flag && 0==level_files.size();
} // if
} // for
return(ret_flag);
} // TestForLevelDirectories
std::string // replacement dbname ... potentially tiered
MakeTieredDbname(
const std::string & dbname, // input ... original dbname from DBImpl constructor
Options & options) // input/output ... writable Options, tiered values changed
{
// case for "", used with internal calls to DestroyDB
if (0==dbname.size() && 0!=options.tiered_fast_prefix.size())
{
// do NOTHING ... options already initialized
} // if
else if (0<(int)options.tiered_slow_level && (int)options.tiered_slow_level<config::kNumLevels
&& 0!=options.tiered_fast_prefix.size() && 0!=options.tiered_slow_prefix.size())
{
options.tiered_fast_prefix.append("/");
options.tiered_fast_prefix.append(dbname);
options.tiered_slow_prefix.append("/");
options.tiered_slow_prefix.append(dbname);
} // else if
else
{
options.tiered_slow_level=0;
options.tiered_fast_prefix=dbname; // duplicate as is
options.tiered_slow_prefix=dbname;
} // else
return(options.tiered_fast_prefix);
} // MakeTieredDbname
} // namespace leveldb
+32 -8
View File
@@ -9,6 +9,7 @@
#include <stdint.h>
#include <string>
#include "leveldb/options.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
#include "port/port.h"
@@ -16,6 +17,7 @@
namespace leveldb {
class Env;
class Version;
enum FileType {
kLogFile,
@@ -24,9 +26,24 @@ enum FileType {
kDescriptorFile,
kCurrentFile,
kTempFile,
kInfoLogFile // Either the current one, or an old one
kInfoLogFile, // Either the current one, or an old one
kCacheWarming
};
// Riak specific routine to help create sst_? subdirectory names
std::string MakeDirName2(const Options & options,
int level, const char* suffix);
// Riak specific routine to help create sst_? subdirectories
Status MakeLevelDirectories(Env * env, const Options & options);
// Riak specific routine to test if sst_? subdirectories exist
bool TestForLevelDirectories(Env * env, const Options & options, class Version *);
// Riak specific routine to standardize conversion of dbname and
// Options' tiered directories (options parameter is MODIFIED)
std::string MakeTieredDbname(const std::string &dbname, Options & options_rw);
// Return the name of the log file with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".
@@ -35,12 +52,8 @@ extern std::string LogFileName(const std::string& dbname, uint64_t number);
// Return the name of the sstable with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".
extern std::string TableFileName(const std::string& dbname, uint64_t number);
// Return the legacy file name for an sstable with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".
extern std::string SSTTableFileName(const std::string& dbname, uint64_t number);
extern std::string TableFileName(const Options & options, uint64_t number,
int level);
// Return the name of the descriptor file for the db named by
// "dbname" and the specified incarnation number. The result will be
@@ -67,10 +80,21 @@ extern std::string InfoLogFileName(const std::string& dbname);
// Return the name of the old info log file for "dbname".
extern std::string OldInfoLogFileName(const std::string& dbname);
// Return the name of the cache object file for the db named by
// "dbname". The result will be prefixed with "dbname".
extern std::string CowFileName(const std::string& dbname);
// Append appropriate "backup" string to input path
extern std::string BackupPath(const std::string& dbname, int backup_num);
// update tiered_fast_prefix and tiered_slow_prefix members of
// given Options object to point to backup path
extern bool SetBackupPaths(Options & options, int backup_num);
// If filename is a leveldb file, store the type of the file in *type.
// The number encoded in the filename is stored in *number. If the
// filename was successfully parsed, returns true. Else return false.
extern bool ParseFileName(const std::string& filename,
extern bool ParseFileName(const std::string& tiered_filename,
uint64_t* number,
FileType* type);
+74 -4
View File
@@ -27,7 +27,6 @@ TEST(FileNameTest, Parse) {
{ "100.log", 100, kLogFile },
{ "0.log", 0, kLogFile },
{ "0.sst", 0, kTableFile },
{ "0.ldb", 0, kTableFile },
{ "CURRENT", 0, kCurrentFile },
{ "LOCK", 0, kDBLockFile },
{ "MANIFEST-2", 2, kDescriptorFile },
@@ -71,13 +70,14 @@ TEST(FileNameTest, Parse) {
for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
std::string f = errors[i];
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
}
};
}
TEST(FileNameTest, Construction) {
uint64_t number;
FileType type;
std::string fname;
Options options;
fname = CurrentFileName("foo");
ASSERT_EQ("foo/", std::string(fname.data(), 4));
@@ -97,12 +97,40 @@ TEST(FileNameTest, Construction) {
ASSERT_EQ(192, number);
ASSERT_EQ(kLogFile, type);
fname = TableFileName("bar", 200);
options.tiered_fast_prefix="bar";
options.tiered_slow_prefix="bar";
fname = TableFileName(options, 200, 1);
ASSERT_EQ("bar/", std::string(fname.data(), 4));
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
ASSERT_EQ("sst_1/", std::string(fname.substr(4,6)));
ASSERT_TRUE(ParseFileName(fname.c_str() + 10, &number, &type));
ASSERT_EQ(200, number);
ASSERT_EQ(kTableFile, type);
fname = TableFileName(options, 400, 4);
ASSERT_EQ("bar/", std::string(fname.data(), 4));
ASSERT_EQ("sst_4/", std::string(fname.substr(4,6)));
ASSERT_TRUE(ParseFileName(fname.c_str() + 10, &number, &type));
ASSERT_EQ(400, number);
ASSERT_EQ(kTableFile, type);
options.tiered_slow_level=4;
options.tiered_fast_prefix="fast";
options.tiered_slow_prefix="slow";
fname = TableFileName(options, 500, 3);
ASSERT_EQ("fast/", std::string(fname.data(), 5));
ASSERT_EQ("sst_3/", std::string(fname.substr(5,6)));
ASSERT_TRUE(ParseFileName(fname.c_str() + 11, &number, &type));
ASSERT_EQ(500, number);
ASSERT_EQ(kTableFile, type);
fname = TableFileName(options, 600, 4);
ASSERT_EQ("slow/", std::string(fname.data(), 5));
ASSERT_EQ("sst_4/", std::string(fname.substr(5,6)));
ASSERT_TRUE(ParseFileName(fname.c_str() + 11, &number, &type));
ASSERT_EQ(600, number);
ASSERT_EQ(kTableFile, type);
fname = DescriptorFileName("bar", 100);
ASSERT_EQ("bar/", std::string(fname.data(), 4));
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
@@ -114,6 +142,48 @@ TEST(FileNameTest, Construction) {
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
ASSERT_EQ(999, number);
ASSERT_EQ(kTempFile, type);
fname = CowFileName("/what/goes/moo");
ASSERT_EQ("/what/goes/moo/COW", fname);
fname = BackupPath("/var/db/riak/data/leveldb/0",0);
ASSERT_EQ("/var/db/riak/data/leveldb/0/backup", fname);
fname = BackupPath("/var/db/riak/data/leveldb/0",1);
ASSERT_EQ("/var/db/riak/data/leveldb/0/backup.1", fname);
fname = BackupPath("/var/db/riak/data/leveldb/0",5);
ASSERT_EQ("/var/db/riak/data/leveldb/0/backup.5", fname);
options.tiered_slow_level=4;
options.tiered_fast_prefix="fast";
options.tiered_slow_prefix="slow";
fname = SetBackupPaths(options,0);
ASSERT_EQ("fast/backup", options.tiered_fast_prefix);
ASSERT_EQ("slow/backup", options.tiered_slow_prefix);
options.tiered_slow_level=4;
options.tiered_fast_prefix="fast";
options.tiered_slow_prefix="slow";
fname = SetBackupPaths(options,3);
ASSERT_EQ("fast/backup.3", options.tiered_fast_prefix);
ASSERT_EQ("slow/backup.3", options.tiered_slow_prefix);
options.tiered_slow_level=4;
options.tiered_fast_prefix="//mnt/fast";
options.tiered_slow_prefix="//mnt/slow";
fname=MakeTieredDbname("riak/data/leveldb", options);
ASSERT_EQ("//mnt/fast/riak/data/leveldb", fname);
ASSERT_EQ("//mnt/fast/riak/data/leveldb", options.tiered_fast_prefix);
ASSERT_EQ("//mnt/slow/riak/data/leveldb", options.tiered_slow_prefix);
// special case with no dbname given, should have no changes
fname=MakeTieredDbname("", options);
ASSERT_EQ("//mnt/fast/riak/data/leveldb", fname);
ASSERT_EQ("//mnt/fast/riak/data/leveldb", options.tiered_fast_prefix);
ASSERT_EQ("//mnt/slow/riak/data/leveldb", options.tiered_slow_prefix);
}
} // namespace leveldb
-65
View File
@@ -1,65 +0,0 @@
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h>
#include "leveldb/dumpfile.h"
#include "leveldb/env.h"
#include "leveldb/status.h"
namespace leveldb {
namespace {
class StdoutPrinter : public WritableFile {
public:
virtual Status Append(const Slice& data) {
fwrite(data.data(), 1, data.size(), stdout);
return Status::OK();
}
virtual Status Close() { return Status::OK(); }
virtual Status Flush() { return Status::OK(); }
virtual Status Sync() { return Status::OK(); }
virtual std::string GetName() const { return "[stdout]"; }
};
bool HandleDumpCommand(Env* env, char** files, int num) {
StdoutPrinter printer;
bool ok = true;
for (int i = 0; i < num; i++) {
Status s = DumpFile(env, files[i], &printer);
if (!s.ok()) {
fprintf(stderr, "%s\n", s.ToString().c_str());
ok = false;
}
}
return ok;
}
} // namespace
} // namespace leveldb
static void Usage() {
fprintf(
stderr,
"Usage: leveldbutil command...\n"
" dump files... -- dump contents of specified files\n"
);
}
int main(int argc, char** argv) {
leveldb::Env* env = leveldb::Env::Default();
bool ok = true;
if (argc < 2) {
Usage();
ok = false;
} else {
std::string command = argv[1];
if (command == "dump") {
ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
} else {
Usage();
ok = false;
}
}
return (ok ? 0 : 1);
}
+3 -3
View File
@@ -3,7 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// Log format information shared by reader and writer.
// See ../doc/log_format.md for more detail.
// See ../doc/log_format.txt for more detail.
#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
@@ -26,8 +26,8 @@ static const int kMaxRecordType = kLastType;
static const int kBlockSize = 32768;
// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
static const int kHeaderSize = 4 + 2 + 1;
// Header is checksum (4 bytes), type (1 byte), length (2 bytes).
static const int kHeaderSize = 4 + 1 + 2;
} // namespace log
} // namespace leveldb
+16 -41
View File
@@ -25,8 +25,7 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
eof_(false),
last_record_offset_(0),
end_of_buffer_offset_(0),
initial_offset_(initial_offset),
resyncing_(initial_offset > 0) {
initial_offset_(initial_offset) {
}
Reader::~Reader() {
@@ -73,25 +72,8 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
Slice fragment;
while (true) {
uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
const unsigned int record_type = ReadPhysicalRecord(&fragment);
// ReadPhysicalRecord may have only had an empty trailer remaining in its
// internal buffer. Calculate the offset of the next physical record now
// that it has returned, properly accounting for its header size.
uint64_t physical_record_offset =
end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();
if (resyncing_) {
if (record_type == kMiddleType) {
continue;
} else if (record_type == kLastType) {
resyncing_ = false;
continue;
} else {
resyncing_ = false;
}
}
switch (record_type) {
case kFullType:
if (in_fragmented_record) {
@@ -151,9 +133,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
case kEof:
if (in_fragmented_record) {
// This can be caused by the writer dying immediately after
// writing a physical record but before completing the next; don't
// treat it as a corruption, just ignore the entire logical record.
ReportCorruption(scratch->size(), "partial record without end(3)");
scratch->clear();
}
return false;
@@ -185,20 +165,20 @@ uint64_t Reader::LastRecordOffset() {
return last_record_offset_;
}
void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
ReportDrop(bytes, Status::Corruption(reason, file_->GetName()));
void Reader::ReportCorruption(size_t bytes, const char* reason) {
ReportDrop(bytes, Status::Corruption(reason));
}
void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
void Reader::ReportDrop(size_t bytes, const Status& reason) {
if (reporter_ != NULL &&
end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
reporter_->Corruption(static_cast<size_t>(bytes), reason);
reporter_->Corruption(bytes, reason);
}
}
unsigned int Reader::ReadPhysicalRecord(Slice* result) {
while (true) {
if (buffer_.size() < kHeaderSize) {
if (buffer_.size() < (size_t)kHeaderSize) {
if (!eof_) {
// Last read was a full read, so this is a trailer to skip
buffer_.clear();
@@ -209,16 +189,17 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
ReportDrop(kBlockSize, status);
eof_ = true;
return kEof;
} else if (buffer_.size() < kBlockSize) {
} else if (buffer_.size() < (size_t)kBlockSize) {
eof_ = true;
}
continue;
} else if (buffer_.size() == 0) {
// End of file
return kEof;
} else {
// Note that if buffer_ is non-empty, we have a truncated header at the
// end of the file, which can be caused by the writer crashing in the
// middle of writing the header. Instead of considering this an error,
// just report EOF.
size_t drop_size = buffer_.size();
buffer_.clear();
ReportCorruption(drop_size, "truncated record at end of file");
return kEof;
}
}
@@ -232,14 +213,8 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
if (kHeaderSize + length > buffer_.size()) {
size_t drop_size = buffer_.size();
buffer_.clear();
if (!eof_) {
ReportCorruption(drop_size, "bad record length");
return kBadRecord;
}
// If the end of the file has been reached without reading |length| bytes
// of payload, assume the writer died in the middle of writing the record.
// Don't report a corruption.
return kEof;
ReportCorruption(drop_size, "bad record length");
return kBadRecord;
}
if (type == kZeroType && length == 0) {
+2 -7
View File
@@ -73,11 +73,6 @@ class Reader {
// Offset at which to start looking for the first record to return
uint64_t const initial_offset_;
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
// particular, a run of kMiddleType and kLastType records can be silently
// skipped in this mode
bool resyncing_;
// Extend record types with the following special values
enum {
kEof = kMaxRecordType + 1,
@@ -99,8 +94,8 @@ class Reader {
// Reports dropped bytes to the reporter.
// buffer_ must be updated to remove the dropped bytes prior to invocation.
void ReportCorruption(uint64_t bytes, const char* reason);
void ReportDrop(uint64_t bytes, const Status& reason);
void ReportCorruption(size_t bytes, const char* reason);
void ReportDrop(size_t bytes, const Status& reason);
// No copying allowed
Reader(const Reader&);
+26 -117
View File
@@ -79,7 +79,7 @@ class LogTest {
virtual Status Skip(uint64_t n) {
if (n > contents_.size()) {
contents_.clear();
return Status::NotFound("in-memory file skipped past end");
return Status::NotFound("in-memory file skipepd past end");
}
contents_.remove_prefix(n);
@@ -104,34 +104,23 @@ class LogTest {
StringSource source_;
ReportCollector report_;
bool reading_;
Writer* writer_;
Reader* reader_;
Writer writer_;
Reader reader_;
// Record metadata for testing initial offset functionality
static size_t initial_offset_record_sizes_[];
static uint64_t initial_offset_last_record_offsets_[];
static int num_initial_offset_records_;
public:
LogTest() : reading_(false),
writer_(new Writer(&dest_)),
reader_(new Reader(&source_, &report_, true/*checksum*/,
0/*initial_offset*/)) {
}
~LogTest() {
delete writer_;
delete reader_;
}
void ReopenForAppend() {
delete writer_;
writer_ = new Writer(&dest_, dest_.contents_.size());
writer_(&dest_),
reader_(&source_, &report_, true/*checksum*/,
0/*initial_offset*/) {
}
void Write(const std::string& msg) {
ASSERT_TRUE(!reading_) << "Write() after starting to read";
writer_->AddRecord(Slice(msg));
writer_.AddRecord(Slice(msg));
}
size_t WrittenBytes() const {
@@ -145,7 +134,7 @@ class LogTest {
}
std::string scratch;
Slice record;
if (reader_->ReadRecord(&record, &scratch)) {
if (reader_.ReadRecord(&record, &scratch)) {
return record.ToString();
} else {
return "EOF";
@@ -193,18 +182,13 @@ class LogTest {
}
void WriteInitialOffsetLog() {
for (int i = 0; i < num_initial_offset_records_; i++) {
for (int i = 0; i < 4; i++) {
std::string record(initial_offset_record_sizes_[i],
static_cast<char>('a' + i));
Write(record);
}
}
void StartReadingAt(uint64_t initial_offset) {
delete reader_;
reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset);
}
void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
WriteInitialOffsetLog();
reading_ = true;
@@ -224,48 +208,32 @@ class LogTest {
source_.contents_ = Slice(dest_.contents_);
Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
initial_offset);
// Read all records from expected_record_offset through the last one.
ASSERT_LT(expected_record_offset, num_initial_offset_records_);
for (; expected_record_offset < num_initial_offset_records_;
++expected_record_offset) {
Slice record;
std::string scratch;
ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
record.size());
ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
offset_reader->LastRecordOffset());
ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
}
Slice record;
std::string scratch;
ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
record.size());
ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
offset_reader->LastRecordOffset());
ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
delete offset_reader;
}
};
size_t LogTest::initial_offset_record_sizes_[] =
{10000, // Two sizable records in first block
10000,
2 * log::kBlockSize - 1000, // Span three blocks
1,
13716, // Consume all but two bytes of block 3.
log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
};
1};
uint64_t LogTest::initial_offset_last_record_offsets_[] =
{0,
kHeaderSize + 10000,
2 * (kHeaderSize + 10000),
2 * (kHeaderSize + 10000) +
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
2 * (kHeaderSize + 10000) +
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize
+ kHeaderSize + 1,
3 * log::kBlockSize,
};
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize};
// LogTest::initial_offset_last_record_offsets_ must be defined before this.
int LogTest::num_initial_offset_records_ =
sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t);
TEST(LogTest, Empty) {
ASSERT_EQ("EOF", Read());
@@ -350,15 +318,6 @@ TEST(LogTest, AlignedEof) {
ASSERT_EQ("EOF", Read());
}
TEST(LogTest, OpenForAppend) {
Write("hello");
ReopenForAppend();
Write("world");
ASSERT_EQ("hello", Read());
ASSERT_EQ("world", Read());
ASSERT_EQ("EOF", Read());
}
TEST(LogTest, RandomRead) {
const int N = 500;
Random write_rnd(301);
@@ -392,32 +351,20 @@ TEST(LogTest, BadRecordType) {
ASSERT_EQ("OK", MatchError("unknown record type"));
}
TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
TEST(LogTest, TruncatedTrailingRecord) {
Write("foo");
ShrinkSize(4); // Drop all payload as well as a header byte
ASSERT_EQ("EOF", Read());
// Truncated last record is ignored, not treated as an error.
ASSERT_EQ(0, DroppedBytes());
ASSERT_EQ("", ReportMessage());
ASSERT_EQ(kHeaderSize - 1, DroppedBytes());
ASSERT_EQ("OK", MatchError("truncated record at end of file"));
}
TEST(LogTest, BadLength) {
const int kPayloadSize = kBlockSize - kHeaderSize;
Write(BigString("bar", kPayloadSize));
Write("foo");
// Least significant size byte is stored in header[4].
IncrementByte(4, 1);
ASSERT_EQ("foo", Read());
ASSERT_EQ(kBlockSize, DroppedBytes());
ASSERT_EQ("OK", MatchError("bad record length"));
}
TEST(LogTest, BadLengthAtEndIsIgnored) {
Write("foo");
ShrinkSize(1);
ASSERT_EQ("EOF", Read());
ASSERT_EQ(0, DroppedBytes());
ASSERT_EQ("", ReportMessage());
ASSERT_EQ(kHeaderSize + 2, DroppedBytes());
ASSERT_EQ("OK", MatchError("bad record length"));
}
TEST(LogTest, ChecksumMismatch) {
@@ -468,40 +415,6 @@ TEST(LogTest, UnexpectedFirstType) {
ASSERT_EQ("OK", MatchError("partial record without end"));
}
TEST(LogTest, MissingLastIsIgnored) {
Write(BigString("bar", kBlockSize));
// Remove the LAST block, including header.
ShrinkSize(14);
ASSERT_EQ("EOF", Read());
ASSERT_EQ("", ReportMessage());
ASSERT_EQ(0, DroppedBytes());
}
TEST(LogTest, PartialLastIsIgnored) {
Write(BigString("bar", kBlockSize));
// Cause a bad record length in the LAST block.
ShrinkSize(1);
ASSERT_EQ("EOF", Read());
ASSERT_EQ("", ReportMessage());
ASSERT_EQ(0, DroppedBytes());
}
TEST(LogTest, SkipIntoMultiRecord) {
// Consider a fragmented record:
// first(R1), middle(R1), last(R1), first(R2)
// If initial_offset points to a record after first(R1) but before first(R2)
// incomplete fragment errors are not actual errors, and must be suppressed
// until a new first or full record is encountered.
Write(BigString("foo", 3*kBlockSize));
Write("correct");
StartReadingAt(kBlockSize);
ASSERT_EQ("correct", Read());
ASSERT_EQ("", ReportMessage());
ASSERT_EQ(0, DroppedBytes());
ASSERT_EQ("EOF", Read());
}
TEST(LogTest, ErrorJoinsRecords) {
// Consider two fragmented records:
// first(R1) last(R1) first(R2) last(R2)
@@ -520,7 +433,7 @@ TEST(LogTest, ErrorJoinsRecords) {
ASSERT_EQ("correct", Read());
ASSERT_EQ("EOF", Read());
const size_t dropped = DroppedBytes();
const int dropped = DroppedBytes();
ASSERT_LE(dropped, 2*kBlockSize + 100);
ASSERT_GE(dropped, 2*kBlockSize);
}
@@ -571,10 +484,6 @@ TEST(LogTest, ReadFourthStart) {
3);
}
TEST(LogTest, ReadInitialOffsetIntoBlockPadding) {
CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
}
TEST(LogTest, ReadEnd) {
CheckOffsetPastEndReturnsNoRecords(0);
}
+5 -14
View File
@@ -12,22 +12,13 @@
namespace leveldb {
namespace log {
static void InitTypeCrc(uint32_t* type_crc) {
for (int i = 0; i <= kMaxRecordType; i++) {
char t = static_cast<char>(i);
type_crc[i] = crc32c::Value(&t, 1);
}
}
Writer::Writer(WritableFile* dest)
: dest_(dest),
block_offset_(0) {
InitTypeCrc(type_crc_);
}
Writer::Writer(WritableFile* dest, uint64_t dest_length)
: dest_(dest), block_offset_(dest_length % kBlockSize) {
InitTypeCrc(type_crc_);
for (int i = 0; i <= kMaxRecordType; i++) {
char t = static_cast<char>(i);
type_crc_[i] = crc32c::Value(&t, 1);
}
}
Writer::~Writer() {
@@ -83,7 +74,7 @@ Status Writer::AddRecord(const Slice& slice) {
Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
assert(n <= 0xffff); // Must fit in two bytes
assert(block_offset_ + kHeaderSize + n <= kBlockSize);
assert(block_offset_ + kHeaderSize + (int)n <= kBlockSize);
// Format the header
char buf[kHeaderSize];
+3 -8
View File
@@ -9,11 +9,10 @@
#include "db/log_format.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
#include "leveldb/env.h"
namespace leveldb {
class WritableFile;
namespace log {
class Writer {
@@ -22,16 +21,12 @@ class Writer {
// "*dest" must be initially empty.
// "*dest" must remain live while this Writer is in use.
explicit Writer(WritableFile* dest);
// Create a writer that will append data to "*dest".
// "*dest" must have initial length "dest_length".
// "*dest" must remain live while this Writer is in use.
Writer(WritableFile* dest, uint64_t dest_length);
~Writer();
Status AddRecord(const Slice& slice);
void Close() {delete dest_; dest_=NULL;};
private:
WritableFile* dest_;
int block_offset_; // Current offset in block
+66 -12
View File
@@ -6,6 +6,7 @@
#include "db/dbformat.h"
#include "leveldb/comparator.h"
#include "leveldb/env.h"
#include "leveldb/expiry.h"
#include "leveldb/iterator.h"
#include "util/coding.h"
@@ -63,6 +64,8 @@ class MemTableIterator: public Iterator {
Slice key_slice = GetLengthPrefixedSlice(iter_.key());
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
}
virtual KeyMetaData & keymetadata() const
{MemTable::DecodeKeyMetaData(iter_.key(), keymetadata_); return(keymetadata_);};
virtual Status status() const { return Status::OK(); }
@@ -81,7 +84,8 @@ Iterator* MemTable::NewIterator() {
void MemTable::Add(SequenceNumber s, ValueType type,
const Slice& key,
const Slice& value) {
const Slice& value,
const ExpiryTimeMicros & expiry) {
// Format of an entry is concatenation of:
// key_size : varint32 of internal_key.size()
// key bytes : char[internal_key.size()]
@@ -89,7 +93,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
// value bytes : char[value.size()]
size_t key_size = key.size();
size_t val_size = value.size();
size_t internal_key_size = key_size + 8;
size_t internal_key_size = key_size + KeySuffixSize(type);
const size_t encoded_len =
VarintLength(internal_key_size) + internal_key_size +
VarintLength(val_size) + val_size;
@@ -97,15 +101,22 @@ void MemTable::Add(SequenceNumber s, ValueType type,
char* p = EncodeVarint32(buf, internal_key_size);
memcpy(p, key.data(), key_size);
p += key_size;
if (IsExpiryKey(type))
{
EncodeFixed64(p, expiry);
p+=8;
}
EncodeFixed64(p, (s << 8) | type);
p += 8;
p = EncodeVarint32(p, val_size);
memcpy(p, value.data(), val_size);
assert(p + val_size == buf + encoded_len);
assert((size_t)((p + val_size) - buf) == encoded_len);
table_.Insert(buf);
}
bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
bool MemTable::Get(const LookupKey& key, Value* value, Status* s,
const Options * options) {
bool ret_flag(false);
Slice memkey = key.memtable_key();
Table::Iterator iter(&table_);
iter.Seek(memkey.data());
@@ -113,6 +124,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
// entry format is:
// klength varint32
// userkey char[klength]
// optional uint64
// tag uint64
// vlength varint32
// value char[vlength]
@@ -122,24 +134,66 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
const char* entry = iter.key();
uint32_t key_length;
const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
Slice internal_key(key_ptr, key_length);
if (comparator_.comparator.user_comparator()->Compare(
Slice(key_ptr, key_length - 8),
ExtractUserKey(internal_key),
key.user_key()) == 0) {
// Correct user key
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
switch (static_cast<ValueType>(tag & 0xff)) {
case kTypeValue: {
KeyMetaData meta;
DecodeKeyMetaData(entry, meta);
switch (meta.m_Type) {
case kTypeValueWriteTime:
case kTypeValueExplicitExpiry:
{
bool expired=false;
if (NULL!=options && options->ExpiryActivated())
expired=options->expiry_module->MemTableCallback(internal_key);
if (expired)
{
// like kTypeDeletion
*s = Status::NotFound(Slice());
ret_flag=true;
break;
} // if
//otherwise fall into kTypeValue code
} // case
case kTypeValue:
{
Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
value->assign(v.data(), v.size());
return true;
ret_flag=true;
break;
}
case kTypeDeletion:
*s = Status::NotFound(Slice());
return true;
}
ret_flag=true;
break;
} // switch
// only unpack metadata if requested
if (key.WantsKeyMetaData())
key.SetKeyMetaData(meta);
}
}
return false;
return ret_flag;
}
// this is a static function
void MemTable::DecodeKeyMetaData(
const char * key,
KeyMetaData & meta)
{
Slice key_slice = GetLengthPrefixedSlice(key);
meta.m_Type=ExtractValueType(key_slice);
meta.m_Sequence=ExtractSequenceNumber(key_slice);
if (IsExpiryKey(meta.m_Type))
meta.m_Expiry=ExtractExpiry(key_slice);
else
meta.m_Expiry=0;
} // DecodeKeyMetaData
} // namespace leveldb
+13 -6
View File
@@ -24,10 +24,10 @@ class MemTable {
explicit MemTable(const InternalKeyComparator& comparator);
// Increase reference count.
void Ref() { ++refs_; }
void Ref() volatile { ++refs_; }
// Drop reference count. Delete if no more references exist.
void Unref() {
void Unref() volatile {
--refs_;
assert(refs_ >= 0);
if (refs_ <= 0) {
@@ -36,7 +36,10 @@ class MemTable {
}
// Returns an estimate of the number of bytes of data in use by this
// data structure. It is safe to call when MemTable is being modified.
// data structure.
//
// REQUIRES: external synchronization to prevent simultaneous
// operations on the same MemTable.
size_t ApproximateMemoryUsage();
// Return an iterator that yields the contents of the memtable.
@@ -52,13 +55,17 @@ class MemTable {
// Typically value will be empty if type==kTypeDeletion.
void Add(SequenceNumber seq, ValueType type,
const Slice& key,
const Slice& value);
const Slice& value,
const ExpiryTimeMicros& expiry=0);
// If memtable contains a value for key, store it in *value and return true.
// If memtable contains a deletion for key, store a NotFound() error
// in *status and return true.
// Else, return false.
bool Get(const LookupKey& key, std::string* value, Status* s);
bool Get(const LookupKey& key, Value* value, Status* s, const Options * options);
// parse keymetadata from skiplist key string
static void DecodeKeyMetaData(const char * key, KeyMetaData & meta);
private:
~MemTable(); // Private since only Unref() should be used to delete it
@@ -69,7 +76,7 @@ class MemTable {
int operator()(const char* a, const char* b) const;
};
friend class MemTableIterator;
friend class MemTableBackwardIterator;
friend class MemTableBackwardIterator; // does not exist
typedef SkipList<const char*, KeyComparator> Table;
+248
View File
@@ -0,0 +1,248 @@
// -------------------------------------------------------------------
//
// penalty_test.cc
//
// Copyright (c) 2016 Basho Technologies, Inc. All Rights Reserved.
//
// This file is provided to you under the Apache License,
// Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
// -------------------------------------------------------------------
#include "util/testharness.h"
#include "util/testutil.h"
#include "leveldb/comparator.h"
#include "db/version_set.h"
/**
* Execution routine
*/
int main(int argc, char** argv)
{
return leveldb::test::RunAllTests();
}
namespace leveldb {
class TestVersion : public Version
{
public:
TestVersion()
: Version(NULL)
{
int loop;
for (loop=0; loop<config::kNumLevels; ++loop)
{
m_FalseFile[loop].file_size=0;
m_LevelFileCount[loop]=0;
} // for
};
virtual size_t NumFiles(int level) const {return(m_LevelFileCount[level]);};
virtual const std::vector<FileMetaData*> & GetFileList(int level) const
{
m_FalseVector.clear();
m_FalseVector.push_back(&m_FalseFile[level]);
return(m_FalseVector);
};
mutable std::vector<FileMetaData*> m_FalseVector;
mutable FileMetaData m_FalseFile[config::kNumLevels];
size_t m_LevelFileCount[config::kNumLevels];
}; // class TestVersion
/**
* Wrapper class for tests. Holds working variables
* and helper functions.
*/
class PenaltyTester : public VersionSet
{
public:
PenaltyTester()
: m_IntCompare(m_Options.comparator), VersionSet("", &m_Options, NULL, &m_IntCompare)
{
};
~PenaltyTester()
{
};
Options m_Options;
InternalKeyComparator m_IntCompare;
}; // class PenaltyTester
/*******************
* Form note:
* using ASSERT_TRUE(0==version.WritePenalty());
* instead of ASSERT_EQ / ASSERT_NE because WritePenalty
* returns a volatile int, which older compilers believe is
* not an equivalent type to a constant. RedHat 5, Solaris,
* and SmartOS were giving grief.
*******************/
/**
* Debug 1
*/
#if 0
TEST(PenaltyTester, Debug1)
{
TestVersion version;
int penalty;
m_Options.write_buffer_size=46416847;
version.m_FalseFile[2].file_size=1075676398;
version.m_LevelFileCount[1]=1;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
} // test Debug1
#endif
/**
* No penalty scenarios
*/
TEST(PenaltyTester, NoPenalty)
{
TestVersion version;
int level;
m_Options.write_buffer_size=46416847;
// nothing
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
/**
* Level 0
* (overlapped level, penalty is count based)
*/
// no penalty
version.m_LevelFileCount[0]=config::kL0_CompactionTrigger;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_LevelFileCount[0]=config::kL0_SlowdownWritesTrigger;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
#if 0 // needs rewrite to be time based
// threshold reached ... some penalty
version.m_LevelFileCount[0]=config::kL0_SlowdownWritesTrigger+1;
UpdatePenalty(&version);
ASSERT_TRUE(0!=version.WritePenalty());
// clean up
version.m_LevelFileCount[0]=0;
/**
* Level 1
* (overlapped level, penalty is count based)
*/
// no penalty
version.m_LevelFileCount[1]=config::kL0_CompactionTrigger;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_LevelFileCount[1]=config::kL0_SlowdownWritesTrigger;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
// threshold reached ... some penalty
version.m_LevelFileCount[1]=config::kL0_SlowdownWritesTrigger+1;
UpdatePenalty(&version);
ASSERT_TRUE(0!=version.WritePenalty());
// clean up
version.m_LevelFileCount[1]=0;
/**
* Level 2
* (landing level, penalty size based)
*/
// no penalty
version.m_FalseFile[2].file_size=0;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_FalseFile[2].file_size=VersionSet::DesiredBytesForLevel(2);
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_FalseFile[2].file_size=VersionSet::MaxBytesForLevel(2)-1;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_FalseFile[2].file_size=VersionSet::MaxBytesForLevel(2);
UpdatePenalty(&version);
ASSERT_TRUE(0!=version.WritePenalty());
// interaction rule with level 1
version.m_FalseFile[2].file_size=VersionSet::MaxBytesForLevel(2)-1;
version.m_LevelFileCount[1]=config::kL0_CompactionTrigger/2;
UpdatePenalty(&version);
ASSERT_TRUE(0!=version.WritePenalty());
// clean up
version.m_LevelFileCount[1]=0;
version.m_FalseFile[2].file_size=0;
/**
* Level 3+
* (landing level, penalty size based)
*/
for (level=3; level<config::kNumLevels; ++level)
{
// no penalty
version.m_FalseFile[level].file_size=0;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_FalseFile[level].file_size=VersionSet::DesiredBytesForLevel(level);
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_FalseFile[level].file_size=VersionSet::MaxBytesForLevel(level)-1;
UpdatePenalty(&version);
ASSERT_TRUE(0==version.WritePenalty());
version.m_FalseFile[level].file_size=VersionSet::MaxBytesForLevel(level);
UpdatePenalty(&version);
if ((config::kNumLevels-1)!=level)
ASSERT_TRUE(0!=version.WritePenalty());
else
ASSERT_TRUE(0==version.WritePenalty());
// clean up
version.m_FalseFile[level].file_size=0;
} // for
#endif
} // test NoPenalty
} // namespace leveldb
-324
View File
@@ -1,324 +0,0 @@
// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/write_batch.h"
#include "util/logging.h"
#include "util/testharness.h"
#include "util/testutil.h"
namespace leveldb {
class RecoveryTest {
public:
RecoveryTest() : env_(Env::Default()), db_(NULL) {
dbname_ = test::TmpDir() + "/recovery_test";
DestroyDB(dbname_, Options());
Open();
}
~RecoveryTest() {
Close();
DestroyDB(dbname_, Options());
}
DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
Env* env() const { return env_; }
bool CanAppend() {
WritableFile* tmp;
Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
delete tmp;
if (s.IsNotSupportedError()) {
return false;
} else {
return true;
}
}
void Close() {
delete db_;
db_ = NULL;
}
void Open(Options* options = NULL) {
Close();
Options opts;
if (options != NULL) {
opts = *options;
} else {
opts.reuse_logs = true; // TODO(sanjay): test both ways
opts.create_if_missing = true;
}
if (opts.env == NULL) {
opts.env = env_;
}
ASSERT_OK(DB::Open(opts, dbname_, &db_));
ASSERT_EQ(1, NumLogs());
}
Status Put(const std::string& k, const std::string& v) {
return db_->Put(WriteOptions(), k, v);
}
std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
std::string result;
Status s = db_->Get(ReadOptions(), k, &result);
if (s.IsNotFound()) {
result = "NOT_FOUND";
} else if (!s.ok()) {
result = s.ToString();
}
return result;
}
std::string ManifestFileName() {
std::string current;
ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), &current));
size_t len = current.size();
if (len > 0 && current[len-1] == '\n') {
current.resize(len - 1);
}
return dbname_ + "/" + current;
}
std::string LogName(uint64_t number) {
return LogFileName(dbname_, number);
}
size_t DeleteLogFiles() {
std::vector<uint64_t> logs = GetFiles(kLogFile);
for (size_t i = 0; i < logs.size(); i++) {
ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
}
return logs.size();
}
uint64_t FirstLogFile() {
return GetFiles(kLogFile)[0];
}
std::vector<uint64_t> GetFiles(FileType t) {
std::vector<std::string> filenames;
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
std::vector<uint64_t> result;
for (size_t i = 0; i < filenames.size(); i++) {
uint64_t number;
FileType type;
if (ParseFileName(filenames[i], &number, &type) && type == t) {
result.push_back(number);
}
}
return result;
}
int NumLogs() {
return GetFiles(kLogFile).size();
}
int NumTables() {
return GetFiles(kTableFile).size();
}
uint64_t FileSize(const std::string& fname) {
uint64_t result;
ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
return result;
}
void CompactMemTable() {
dbfull()->TEST_CompactMemTable();
}
// Directly construct a log file that sets key to val.
void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
std::string fname = LogFileName(dbname_, lognum);
WritableFile* file;
ASSERT_OK(env_->NewWritableFile(fname, &file));
log::Writer writer(file);
WriteBatch batch;
batch.Put(key, val);
WriteBatchInternal::SetSequence(&batch, seq);
ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
ASSERT_OK(file->Flush());
delete file;
}
private:
std::string dbname_;
Env* env_;
DB* db_;
};
TEST(RecoveryTest, ManifestReused) {
if (!CanAppend()) {
fprintf(stderr, "skipping test because env does not support appending\n");
return;
}
ASSERT_OK(Put("foo", "bar"));
Close();
std::string old_manifest = ManifestFileName();
Open();
ASSERT_EQ(old_manifest, ManifestFileName());
ASSERT_EQ("bar", Get("foo"));
Open();
ASSERT_EQ(old_manifest, ManifestFileName());
ASSERT_EQ("bar", Get("foo"));
}
TEST(RecoveryTest, LargeManifestCompacted) {
if (!CanAppend()) {
fprintf(stderr, "skipping test because env does not support appending\n");
return;
}
ASSERT_OK(Put("foo", "bar"));
Close();
std::string old_manifest = ManifestFileName();
// Pad with zeroes to make manifest file very big.
{
uint64_t len = FileSize(old_manifest);
WritableFile* file;
ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
std::string zeroes(3*1048576 - static_cast<size_t>(len), 0);
ASSERT_OK(file->Append(zeroes));
ASSERT_OK(file->Flush());
delete file;
}
Open();
std::string new_manifest = ManifestFileName();
ASSERT_NE(old_manifest, new_manifest);
ASSERT_GT(10000, FileSize(new_manifest));
ASSERT_EQ("bar", Get("foo"));
Open();
ASSERT_EQ(new_manifest, ManifestFileName());
ASSERT_EQ("bar", Get("foo"));
}
TEST(RecoveryTest, NoLogFiles) {
ASSERT_OK(Put("foo", "bar"));
ASSERT_EQ(1, DeleteLogFiles());
Open();
ASSERT_EQ("NOT_FOUND", Get("foo"));
Open();
ASSERT_EQ("NOT_FOUND", Get("foo"));
}
TEST(RecoveryTest, LogFileReuse) {
if (!CanAppend()) {
fprintf(stderr, "skipping test because env does not support appending\n");
return;
}
for (int i = 0; i < 2; i++) {
ASSERT_OK(Put("foo", "bar"));
if (i == 0) {
// Compact to ensure current log is empty
CompactMemTable();
}
Close();
ASSERT_EQ(1, NumLogs());
uint64_t number = FirstLogFile();
if (i == 0) {
ASSERT_EQ(0, FileSize(LogName(number)));
} else {
ASSERT_LT(0, FileSize(LogName(number)));
}
Open();
ASSERT_EQ(1, NumLogs());
ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
ASSERT_EQ("bar", Get("foo"));
Open();
ASSERT_EQ(1, NumLogs());
ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
ASSERT_EQ("bar", Get("foo"));
}
}
TEST(RecoveryTest, MultipleMemTables) {
// Make a large log.
const int kNum = 1000;
for (int i = 0; i < kNum; i++) {
char buf[100];
snprintf(buf, sizeof(buf), "%050d", i);
ASSERT_OK(Put(buf, buf));
}
ASSERT_EQ(0, NumTables());
Close();
ASSERT_EQ(0, NumTables());
ASSERT_EQ(1, NumLogs());
uint64_t old_log_file = FirstLogFile();
// Force creation of multiple memtables by reducing the write buffer size.
Options opt;
opt.reuse_logs = true;
opt.write_buffer_size = (kNum*100) / 2;
Open(&opt);
ASSERT_LE(2, NumTables());
ASSERT_EQ(1, NumLogs());
ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
for (int i = 0; i < kNum; i++) {
char buf[100];
snprintf(buf, sizeof(buf), "%050d", i);
ASSERT_EQ(buf, Get(buf));
}
}
TEST(RecoveryTest, MultipleLogFiles) {
ASSERT_OK(Put("foo", "bar"));
Close();
ASSERT_EQ(1, NumLogs());
// Make a bunch of uncompacted log files.
uint64_t old_log = FirstLogFile();
MakeLogFile(old_log+1, 1000, "hello", "world");
MakeLogFile(old_log+2, 1001, "hi", "there");
MakeLogFile(old_log+3, 1002, "foo", "bar2");
// Recover and check that all log files were processed.
Open();
ASSERT_LE(1, NumTables());
ASSERT_EQ(1, NumLogs());
uint64_t new_log = FirstLogFile();
ASSERT_LE(old_log+3, new_log);
ASSERT_EQ("bar2", Get("foo"));
ASSERT_EQ("world", Get("hello"));
ASSERT_EQ("there", Get("hi"));
// Test that previous recovery produced recoverable state.
Open();
ASSERT_LE(1, NumTables());
ASSERT_EQ(1, NumLogs());
if (CanAppend()) {
ASSERT_EQ(new_log, FirstLogFile());
}
ASSERT_EQ("bar2", Get("foo"));
ASSERT_EQ("world", Get("hello"));
ASSERT_EQ("there", Get("hi"));
// Check that introducing an older log file does not cause it to be re-read.
Close();
MakeLogFile(old_log+1, 2000, "hello", "stale write");
Open();
ASSERT_LE(1, NumTables());
ASSERT_EQ(1, NumLogs());
if (CanAppend()) {
ASSERT_EQ(new_log, FirstLogFile());
}
ASSERT_EQ("bar2", Get("foo"));
ASSERT_EQ("world", Get("hello"));
ASSERT_EQ("there", Get("hi"));
}
} // namespace leveldb
int main(int argc, char** argv) {
return leveldb::test::RunAllTests();
}
+245 -181
View File
@@ -45,49 +45,113 @@ namespace {
class Repairer {
public:
Repairer(const std::string& dbname, const Options& options)
: dbname_(dbname),
: double_cache_(options),
options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options, double_cache_.GetBlockCache())),
org_options_(options),
dbname_(options_.tiered_fast_prefix),
org_dbname_(dbname),
env_(options.env),
icmp_(options.comparator),
ipolicy_(options.filter_policy),
options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
owns_info_log_(options_.info_log != options.info_log),
owns_cache_(options_.block_cache != options.block_cache),
next_file_number_(1) {
db_lock_(NULL),
next_file_number_(1)
{
// TableCache can be small since we expect each table to be opened once.
table_cache_ = new TableCache(dbname_, &options_, 10);
table_cache_ = new TableCache(dbname_, &options_, double_cache_.GetFileCache(), double_cache_);
}
~Repairer() {
delete table_cache_;
if (owns_info_log_) {
delete options_.info_log;
}
if (owns_cache_) {
delete options_.block_cache;
}
// if (owns_cache_) {
// delete options_.block_cache;
// }
// must remove second ref counter that keeps overlapped files locked
// table cache
bool is_overlap;
for (int level = 0; level < config::kNumLevels; level++) {
{
is_overlap=(level < leveldb::config::kNumOverlapLevels);
for (size_t i = 0; i < table_numbers_[level].size(); i++) {
table_cache_->Evict(table_numbers_[level][i], is_overlap);
} // for
} // if
} // for
delete table_cache_;
}
Status Run() {
Status status = FindFiles();
Status status;
status = env_->LockFile(LockFileName(dbname_), &db_lock_);
if (status.ok())
status = MakeLevelDirectories(env_, options_);
if (status.ok()) {
ConvertLogFilesToTables();
ExtractMetaData();
status = WriteDescriptor();
}
if (status.ok()) {
unsigned long long bytes = 0;
for (size_t i = 0; i < tables_.size(); i++) {
bytes += tables_[i].meta.file_size;
status = FindFiles();
if (status.ok()) {
ConvertLogFilesToTables();
ExtractMetaData();
status = WriteDescriptor();
}
if (status.ok()) {
unsigned long long bytes = 0;
unsigned long long files = 0;
// calculate size for log information
for (int level=0; level<config::kNumLevels;++level)
{
std::vector<TableInfo> * table_ptr;
std::vector<TableInfo>::const_iterator i;
table_ptr=&tables_[level];
files+=table_ptr->size();
for ( i = table_ptr->begin(); table_ptr->end()!= i; i++) {
bytes += i->meta.file_size;
}
} // for
Log(options_.info_log,
"**** Repaired leveldb %s; "
"recovered %d files; %llu bytes. "
"Some data may have been lost. "
"****",
dbname_.c_str(),
static_cast<int>(files),
bytes);
}
if (db_lock_ != NULL) {
env_->UnlockFile(db_lock_);
}
Log(options_.info_log,
"**** Repaired leveldb %s; "
"recovered %d files; %llu bytes. "
"Some data may have been lost. "
"****",
dbname_.c_str(),
static_cast<int>(tables_.size()),
bytes);
}
// perform Riak specific scan for overlapping .sst files
// within a level
if (status.ok())
{
leveldb::DB * db_ptr;
Options options;
db_ptr=NULL;
options=org_options_;
// options.block_cache=NULL; // not reusing for fear of edge cases
options.is_repair=true;
options.error_if_exists=false;
status=leveldb::DB::Open(options, org_dbname_, &db_ptr);
if (status.ok())
status=db_ptr->VerifyLevels();
delete db_ptr;
} // if
return status;
}
@@ -97,34 +161,36 @@ class Repairer {
SequenceNumber max_sequence;
};
std::string const dbname_;
DoubleCache double_cache_;
Options const options_, org_options_;
std::string const dbname_, org_dbname_;
Env* const env_;
InternalKeyComparator const icmp_;
InternalFilterPolicy const ipolicy_;
Options const options_;
bool owns_info_log_;
bool owns_cache_;
FileLock* db_lock_;
TableCache* table_cache_;
VersionEdit edit_;
std::vector<std::string> manifests_;
std::vector<uint64_t> table_numbers_;
std::vector<uint64_t> table_numbers_[config::kNumLevels];
std::vector<uint64_t> logs_;
std::vector<TableInfo> tables_;
std::vector<TableInfo> tables_[config::kNumLevels];
uint64_t next_file_number_;
Status FindFiles() {
Status FindFiles()
{
std::vector<std::string> filenames;
uint64_t number;
FileType type;
int level;
// base directory
Status status = env_->GetChildren(dbname_, &filenames);
if (!status.ok()) {
return status;
}
if (filenames.empty()) {
return Status::IOError(dbname_, "repair found no files");
}
uint64_t number;
FileType type;
for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type)) {
if (type == kDescriptorFile) {
@@ -136,13 +202,38 @@ class Repairer {
if (type == kLogFile) {
logs_.push_back(number);
} else if (type == kTableFile) {
table_numbers_.push_back(number);
table_numbers_[0].push_back(number);
} else {
// Ignore other files
}
}
} // else
} // else
} // if
} // for
for (level=0; level < config::kNumLevels; ++level)
{
std::string dirname;
filenames.clear();
dirname=MakeDirName2(options_, level, "sst");
Status status = env_->GetChildren(dirname, &filenames);
if (!status.ok()) {
return status;
}
}
for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type)) {
if (number + 1 > next_file_number_) {
next_file_number_ = number + 1;
}
if (type == kTableFile) {
table_numbers_[level].push_back(number);
}
} // if
} // for
} // for
return status;
}
@@ -186,7 +277,7 @@ class Repairer {
reporter.env = env_;
reporter.info_log = options_.info_log;
reporter.lognum = log;
// We intentionally make log::Reader do checksumming so that
// We intentially make log::Reader do checksumming so that
// corruptions cause entire commits to be skipped instead of
// propagating bad information (like overly large sequence
// numbers).
@@ -203,11 +294,11 @@ class Repairer {
while (reader.ReadRecord(&record, &scratch)) {
if (record.size() < 12) {
reporter.Corruption(
record.size(), Status::Corruption("log record too small", logname));
record.size(), Status::Corruption("log record too small"));
continue;
}
WriteBatchInternal::SetContents(&batch, record);
status = WriteBatchInternal::InsertInto(&batch, mem);
status = WriteBatchInternal::InsertInto(&batch, mem, &options_);
if (status.ok()) {
counter += WriteBatchInternal::Count(&batch);
} else {
@@ -223,14 +314,15 @@ class Repairer {
// since ExtractMetaData() will also generate edits.
FileMetaData meta;
meta.number = next_file_number_++;
meta.level = 0;
Iterator* iter = mem->NewIterator();
status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
status = BuildTable(dbname_, env_, options_, icmp_.user_comparator(), table_cache_, iter, &meta, 0);
delete iter;
mem->Unref();
mem = NULL;
if (status.ok()) {
if (meta.file_size > 0) {
table_numbers_.push_back(meta.number);
table_numbers_[0].push_back(meta.number);
}
}
Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
@@ -242,168 +334,128 @@ class Repairer {
}
void ExtractMetaData() {
for (size_t i = 0; i < table_numbers_.size(); i++) {
ScanTable(table_numbers_[i]);
}
}
for (int level=0; level < config::kNumLevels; ++level)
{
std::vector<uint64_t> * number_ptr;
std::vector<uint64_t>::const_iterator i;
Iterator* NewTableIterator(const FileMetaData& meta) {
// Same as compaction iterators: if paranoid_checks are on, turn
// on checksum verification.
ReadOptions r;
r.verify_checksums = options_.paranoid_checks;
return table_cache_->NewIterator(r, meta.number, meta.file_size);
}
void ScanTable(uint64_t number) {
TableInfo t;
t.meta.number = number;
std::string fname = TableFileName(dbname_, number);
Status status = env_->GetFileSize(fname, &t.meta.file_size);
if (!status.ok()) {
// Try alternate file name.
fname = SSTTableFileName(dbname_, number);
Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
if (s2.ok()) {
status = Status::OK();
number_ptr=&table_numbers_[level];
for (i = number_ptr->begin(); number_ptr->end()!= i; ++i) {
TableInfo t;
t.meta.number = *i;
t.meta.level = level;
Status status = ScanTable(&t);
if (!status.ok())
{
std::string fname = TableFileName(options_, t.meta.number, t.meta.level);
Log(options_.info_log, "Table #%llu: ignoring %s",
(unsigned long long) t.meta.number,
status.ToString().c_str());
ArchiveFile(fname, true);
} else {
tables_[level].push_back(t);
}
}
}
if (!status.ok()) {
ArchiveFile(TableFileName(dbname_, number));
ArchiveFile(SSTTableFileName(dbname_, number));
Log(options_.info_log, "Table #%llu: dropped: %s",
(unsigned long long) t.meta.number,
status.ToString().c_str());
return;
}
}
// Extract metadata by scanning through table.
Status ScanTable(TableInfo* t) {
Table * table_ptr;
SstCounters counters;
std::string fname = TableFileName(options_, t->meta.number, t->meta.level);
int counter = 0;
Iterator* iter = NewTableIterator(t.meta);
bool empty = true;
ParsedInternalKey parsed;
t.max_sequence = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
Slice key = iter->key();
if (!ParseInternalKey(key, &parsed)) {
Log(options_.info_log, "Table #%llu: unparsable key %s",
(unsigned long long) t.meta.number,
EscapeString(key).c_str());
continue;
}
Status status = env_->GetFileSize(fname, &t->meta.file_size);
if (status.ok()) {
Iterator* iter = table_cache_->NewIterator(
ReadOptions(), t->meta.number, t->meta.file_size, t->meta.level, &table_ptr);
bool empty = true;
ParsedInternalKey parsed;
t->max_sequence = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
Slice key = iter->key();
if (!ParseInternalKey(key, &parsed)) {
Log(options_.info_log, "Table #%llu: unparsable key %s",
(unsigned long long) t->meta.number,
EscapeString(key).c_str());
continue;
}
counter++;
if (empty) {
empty = false;
t.meta.smallest.DecodeFrom(key);
counter++;
if (empty) {
empty = false;
t->meta.smallest.DecodeFrom(key);
}
t->meta.largest.DecodeFrom(key);
if (parsed.sequence > t->max_sequence) {
t->max_sequence = parsed.sequence;
}
}
t.meta.largest.DecodeFrom(key);
if (parsed.sequence > t.max_sequence) {
t.max_sequence = parsed.sequence;
if (!iter->status().ok()) {
status = iter->status();
}
else {
counters=table_ptr->GetSstCounters();
t->meta.exp_write_low=counters.Value(eSstCountExpiry1);
t->meta.exp_write_high=counters.Value(eSstCountExpiry2);
t->meta.exp_explicit_high=counters.Value(eSstCountExpiry3);
}
delete iter;
}
if (!iter->status().ok()) {
status = iter->status();
}
delete iter;
Log(options_.info_log, "Table #%llu: %d entries %s",
(unsigned long long) t.meta.number,
(unsigned long long) t->meta.number,
counter,
status.ToString().c_str());
if (status.ok()) {
tables_.push_back(t);
} else {
RepairTable(fname, t); // RepairTable archives input file.
}
}
void RepairTable(const std::string& src, TableInfo t) {
// We will copy src contents to a new table and then rename the
// new table over the source.
// Create builder.
std::string copy = TableFileName(dbname_, next_file_number_++);
WritableFile* file;
Status s = env_->NewWritableFile(copy, &file);
if (!s.ok()) {
return;
}
TableBuilder* builder = new TableBuilder(options_, file);
// Copy data.
Iterator* iter = NewTableIterator(t.meta);
int counter = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
builder->Add(iter->key(), iter->value());
counter++;
}
delete iter;
ArchiveFile(src);
if (counter == 0) {
builder->Abandon(); // Nothing to save
} else {
s = builder->Finish();
if (s.ok()) {
t.meta.file_size = builder->FileSize();
}
}
delete builder;
builder = NULL;
if (s.ok()) {
s = file->Close();
}
delete file;
file = NULL;
if (counter > 0 && s.ok()) {
std::string orig = TableFileName(dbname_, t.meta.number);
s = env_->RenameFile(copy, orig);
if (s.ok()) {
Log(options_.info_log, "Table #%llu: %d entries repaired",
(unsigned long long) t.meta.number, counter);
tables_.push_back(t);
}
}
if (!s.ok()) {
env_->DeleteFile(copy);
}
return status;
}
Status WriteDescriptor() {
std::string tmp = TempFileName(dbname_, 1);
WritableFile* file;
Status status = env_->NewWritableFile(tmp, &file);
Status status = env_->NewWritableFile(tmp, &file, 4096);
if (!status.ok()) {
return status;
}
SequenceNumber max_sequence = 0;
for (size_t i = 0; i < tables_.size(); i++) {
if (max_sequence < tables_[i].max_sequence) {
max_sequence = tables_[i].max_sequence;
}
}
for (int level=0; level<config::kNumLevels;++level)
{
std::vector<TableInfo> * table_ptr;
std::vector<TableInfo>::const_iterator i;
table_ptr=&tables_[level];
for ( i = table_ptr->begin(); table_ptr->end()!= i; i++) {
if (max_sequence < i->max_sequence) {
max_sequence = i->max_sequence;
}
} // for
} // for
edit_.SetComparatorName(icmp_.user_comparator()->Name());
edit_.SetLogNumber(0);
edit_.SetNextFile(next_file_number_);
edit_.SetLastSequence(max_sequence);
for (size_t i = 0; i < tables_.size(); i++) {
// TODO(opt): separate out into multiple levels
const TableInfo& t = tables_[i];
edit_.AddFile(0, t.meta.number, t.meta.file_size,
t.meta.smallest, t.meta.largest);
}
for (int level=0; level<config::kNumLevels;++level)
{
std::vector<TableInfo> * table_ptr;
std::vector<TableInfo>::const_iterator i;
table_ptr=&tables_[level];
for ( i = table_ptr->begin(); table_ptr->end()!= i; i++) {
edit_.AddFile2(level, i->meta.number, i->meta.file_size,
i->meta.smallest, i->meta.largest,
i->meta.exp_write_low, i->meta.exp_write_high, i->meta.exp_explicit_high);
} // for
} // for
//fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
{
log::Writer log(file);
std::string record;
edit_.EncodeTo(&record);
edit_.EncodeTo(&record); // manifest format is default for release, options_ often incomplete
status = log.AddRecord(record);
}
if (status.ok()) {
@@ -431,21 +483,33 @@ class Repairer {
return status;
}
void ArchiveFile(const std::string& fname) {
void ArchiveFile(const std::string& fname, bool two_levels=false) {
// Move into another directory. E.g., for
// dir/foo
// rename to
// dir/lost/foo
const char* slash = strrchr(fname.c_str(), '/');
std::string::size_type slash, slash2;
slash=fname.rfind('/');
if (two_levels && std::string::npos!=slash && 0<slash)
{
slash2=fname.rfind('/',slash-1);
if (std::string::npos==slash2)
slash2=slash;
} // if
else
slash2=slash;
std::string new_dir;
if (slash != NULL) {
new_dir.assign(fname.data(), slash - fname.data());
}
if (std::string::npos != slash2 && 0<slash2)
new_dir.append(fname,0,slash2);
new_dir.append("/lost");
env_->CreateDir(new_dir); // Ignore error
std::string new_file = new_dir;
new_file.append("/");
new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
new_file.append((std::string::npos!=slash) ? fname.substr(slash+1) : fname);
Status s = env_->RenameFile(fname, new_file);
Log(options_.info_log, "Archiving %s: %s\n",
fname.c_str(), s.ToString().c_str());
+245 -13
View File
@@ -1,10 +1,7 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
#define STORAGE_LEVELDB_DB_SKIPLIST_H_
//
// Thread safety
// -------------
//
@@ -55,6 +52,12 @@ class SkipList {
// Returns true iff an entry that compares equal to key is in the list.
bool Contains(const Key& key) const;
// Returns true if all inserts have been sequentially increasing;
// else this SkipList has had keys inserted in non-sequential order
bool InSequentialInsertMode() const {
return sequentialInsertMode_;
}
// Iteration over the contents of a skip list
class Iterator {
public:
@@ -94,8 +97,22 @@ class SkipList {
// Intentionally copyable
};
protected:
// Checks the structure of this SkipList object, ensuring the keys are
// properly ordered
//
// This is protected since it is intended for use by unit tests; if a lock
// is used to protect Insert(), then it should be used to protect this
// method as well
bool Valid() const;
// Disables the sequential insert optimizations (used in performance testing)
void DisableSequentialInsertMode() {
sequentialInsertMode_ = false;
}
private:
enum { kMaxHeight = 12 };
enum { kMaxHeight = 17 };
// Immutable after construction
Comparator const compare_;
@@ -115,6 +132,18 @@ class SkipList {
// Read/written only by Insert().
Random rnd_;
// Points to the last node in the list; modified only by Insert()
Node* tail_;
// Pointers to the nodes previous to the tail node; have max_height_ entries
Node* tailPrev_[kMaxHeight];
// The height of the tail_ node
int tailHeight_;
// We track the tail node until we have a non-sequential insert
bool sequentialInsertMode_;
Node* NewNode(const Key& key, int height);
int RandomHeight();
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
@@ -129,6 +158,11 @@ class SkipList {
// node at "level" for every level in [0..max_height_-1].
Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
// Similar to FindGreaterOrEqual() except it uses the barrier-free
// variant of Next(); this is used only by Insert() and it
// checks the tail_ pointer in case we're doing a sequential insert
Node* NoBarrier_FindGreaterOrEqual(const Key& key, Node** prev) const;
// Return the latest node with a key < key.
// Return head_ if there is no such node.
Node* FindLessThan(const Key& key) const;
@@ -280,6 +314,54 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
}
}
template<typename Key, class Comparator>
typename SkipList<Key,Comparator>::Node*
SkipList<Key,Comparator>::NoBarrier_FindGreaterOrEqual(const Key& key, Node** prev) const {
int level = GetMaxHeight() - 1;
// If we have only seen sequential inserts up to this point, we can use
// the tail_ node
if ( sequentialInsertMode_ ) {
if (tail_ == NULL) {
// The list is currently empty, so the node being inserted
// will be the new tail_
assert(level == 0);
if (prev != NULL) prev[0] = head_;
return NULL;
}
else if (KeyIsAfterNode(key, tail_)) {
// The new key must be inserted after the current tail_ node
if (prev != NULL) {
int i;
for (i = 0; i < tailHeight_; ++i) {
prev[i] = tail_;
}
for (/*continue with i*/; i <= level; ++i) {
prev[i] = tailPrev_[i];
}
}
return NULL;
}
}
Node* x = head_;
while (true) {
Node* next = x->NoBarrier_Next(level);
if (KeyIsAfterNode(key, next)) {
// Keep searching in this list
x = next;
} else {
if (prev != NULL) prev[level] = x;
if (level == 0) {
return next;
} else {
// Switch to next list
level--;
}
}
}
}
template<typename Key, class Comparator>
typename SkipList<Key,Comparator>::Node*
SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
@@ -327,25 +409,41 @@ SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
arena_(arena),
head_(NewNode(0 /* any key will do */, kMaxHeight)),
max_height_(reinterpret_cast<void*>(1)),
rnd_(0xdeadbeef) {
rnd_(0xdeadbeef),
tail_(NULL),
tailHeight_(0),
sequentialInsertMode_(true) {
for (int i = 0; i < kMaxHeight; i++) {
head_->SetNext(i, NULL);
tailPrev_[i] = NULL;
}
}
template<typename Key, class Comparator>
void SkipList<Key,Comparator>::Insert(const Key& key) {
// TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
// We use a barrier-free variant of FindGreaterOrEqual()
// here since Insert() is externally synchronized.
Node* prev[kMaxHeight];
Node* x = FindGreaterOrEqual(key, prev);
Node* x = NoBarrier_FindGreaterOrEqual(key, prev);
// If we're still in sequential-insert mode, check if the new node is being
// inserted at the end of the list, which is indicated by x being NULL
if (sequentialInsertMode_) {
if (x != NULL) {
// we have a non-sequential (AKA random) insert, so stop maintaining
// the tail bookkeeping overhead
sequentialInsertMode_ = false;
}
}
// Our data structure does not allow duplicate insertion
assert(x == NULL || !Equal(key, x->key));
int height = RandomHeight();
int i, height = RandomHeight();
if (height > GetMaxHeight()) {
for (int i = GetMaxHeight(); i < height; i++) {
// We are extending max_height_ which means we need to fill in the blanks
// in prev[] that were not filled in by NoBarrier_FindGreaterOrEqual()
for (i = GetMaxHeight(); i < height; ++i) {
prev[i] = head_;
}
//fprintf(stderr, "Change height from %d to %d\n", max_height_, height);
@@ -361,12 +459,37 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
}
x = NewNode(key, height);
for (int i = 0; i < height; i++) {
for (i = 0; i < height; ++i) {
// NoBarrier_SetNext() suffices since we will add a barrier when
// we publish a pointer to "x" in prev[i].
x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i));
prev[i]->SetNext(i, x);
}
// Do we need to update our tail_ pointer?
if (sequentialInsertMode_) {
Node* prevTail = tail_;
int prevTailHeight = tailHeight_;
tail_ = x;
tailHeight_ = height;
// We also need to update our tailPrev_ pointers; first we capture
// the nodes already pointing to the new tail_
for (i = 0; i < height; ++i) {
tailPrev_[i] = prev[i];
}
// If the previous tail node was taller than the new tail node, then
// the prev pointers above the current tail node's height (up to the
// height of the previous tail node) are simply the previous tail node
for (/*continue with i*/; i < prevTailHeight; ++i) {
tailPrev_[i] = prevTail;
}
// NOTE: any prev pointers above prevTailHeight (up to max_height_) were
// already set in tailPrev_ by previous calls to this method
}
}
template<typename Key, class Comparator>
@@ -379,6 +502,115 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
}
}
} // namespace leveldb
template<typename Key, class Comparator>
bool SkipList<Key,Comparator>::Valid() const
{
// Note that we can use barrier-free overloads in this method since it is
// protected by the same lock as Insert().
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
// Ensure that the list is properly sorted; use an iterator for this check
const Key* pPrevKey = NULL;
typename SkipList<Key, Comparator>::Iterator iter(this);
for ( iter.SeekToFirst(); iter.Valid(); iter.Next() ) {
if ( pPrevKey != NULL ) {
if ( compare_( *pPrevKey, iter.key() ) >= 0 ) {
return false;
}
}
pPrevKey = &iter.key();
}
// Now walk the linked list at each level and ensure it's sorted. Also track
// how many nodes we see at each level; the number of nodes in the linked
// list at level n must not be larger than the number of nodes at level n-1.
std::vector<int> nodeCounts( GetMaxHeight() );
int level;
for ( level = GetMaxHeight() - 1; level >= 0; --level ) {
int nodeCount = 0;
pPrevKey = NULL;
for ( Node* pNode = head_->NoBarrier_Next( level );
pNode != NULL;
pNode = pNode->NoBarrier_Next( level ) ) {
++nodeCount;
if ( pPrevKey != NULL ) {
if ( compare_( *pPrevKey, pNode->key ) >= 0 ) {
return false;
}
}
pPrevKey = &pNode->key;
}
nodeCounts[ level ] = nodeCount;
}
// Ensure the node counts do not increase as we move up the levels
int prevNodeCount = nodeCounts[0];
for ( level = 1; level < GetMaxHeight(); ++level ) {
int currentNodeCount = nodeCounts[ level ];
if ( currentNodeCount > prevNodeCount ) {
return false;
}
prevNodeCount = currentNodeCount;
}
// Ensure that tail_ points to the last node
if ( sequentialInsertMode_ ) {
if ( tail_ == NULL ) {
// tail_ is not set, so the list must be empty
if ( tailPrev_[0] != NULL || head_->NoBarrier_Next(0) != NULL ) {
return false;
}
}
else {
// we have a tail_ node; first ensure that its prev pointer actually
// points to it
if ( tailPrev_[0] == NULL || tailPrev_[0]->NoBarrier_Next(0) != tail_ ) {
return false;
}
if ( compare_( tailPrev_[0]->key, tail_->key ) >= 0 ) {
return false;
}
// now check the rest of the pointers in tailPrev_; up to tailHeight_,
// the next pointer of the node in tailPrev_ should point to tail_; after
// that, the next pointer should be NULL
for ( level = 1; level < GetMaxHeight(); ++level ) {
Node* tailPrev = tailPrev_[ level ];
if ( tailPrev == NULL ) {
return false;
}
if ( level < tailHeight_ ) {
if ( tailPrev->NoBarrier_Next( level ) != tail_ ) {
return false;
}
if ( compare_( tailPrev->key, tail_->key ) >= 0 ) {
return false;
}
}
else {
if ( tailPrev->NoBarrier_Next( level ) != NULL ) {
return false;
}
}
}
// the remainder of the tailPrev_ pointers (above max_height_)
// should be NULL
for ( /*continue with level*/; level < kMaxHeight; ++level ) {
if ( tailPrev_[ level ] != NULL ) {
return false;
}
}
// now ensure that FindLast() returns tail_
Node* lastNode = FindLast();
if ( lastNode != tail_ ) {
return false;
}
}
}
// if we get here, all is good
return true;
}
} // namespace leveldb
+233 -11
View File
@@ -2,11 +2,15 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include "db/skiplist.h"
#include <set>
#include "leveldb/env.h"
#include "util/arena.h"
#include "util/hash.h"
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/testharness.h"
@@ -26,15 +30,29 @@ struct Comparator {
}
};
template<typename Key, class Comparator>
class SkipListTest : public SkipList<Key, Comparator>
{
public:
SkipListTest(Comparator cmp, Arena* arena) : SkipList<Key, Comparator>(cmp, arena) {}
// check the validity of this SkipList object by calling the Valid() method
// in the base class
bool Valid() const { return SkipList<Key, Comparator>::Valid(); }
void DisableSequentialInsertMode() { SkipList<Key, Comparator>::DisableSequentialInsertMode(); }
};
class SkipTest { };
TEST(SkipTest, Empty) {
Arena arena;
Comparator cmp;
SkipList<Key, Comparator> list(cmp, &arena);
SkipListTest<Key, Comparator> list(cmp, &arena);
ASSERT_TRUE(!list.Contains(10));
ASSERT_TRUE(list.Valid());
SkipList<Key, Comparator>::Iterator iter(&list);
SkipListTest<Key, Comparator>::Iterator iter(&list);
ASSERT_TRUE(!iter.Valid());
iter.SeekToFirst();
ASSERT_TRUE(!iter.Valid());
@@ -51,13 +69,14 @@ TEST(SkipTest, InsertAndLookup) {
std::set<Key> keys;
Arena arena;
Comparator cmp;
SkipList<Key, Comparator> list(cmp, &arena);
SkipListTest<Key, Comparator> list(cmp, &arena);
for (int i = 0; i < N; i++) {
Key key = rnd.Next() % R;
if (keys.insert(key).second) {
list.Insert(key);
}
}
ASSERT_TRUE(list.Valid());
for (int i = 0; i < R; i++) {
if (list.Contains(i)) {
@@ -69,7 +88,7 @@ TEST(SkipTest, InsertAndLookup) {
// Simple iterator tests
{
SkipList<Key, Comparator>::Iterator iter(&list);
SkipListTest<Key, Comparator>::Iterator iter(&list);
ASSERT_TRUE(!iter.Valid());
iter.Seek(0);
@@ -87,7 +106,7 @@ TEST(SkipTest, InsertAndLookup) {
// Forward iteration test
for (int i = 0; i < R; i++) {
SkipList<Key, Comparator>::Iterator iter(&list);
SkipListTest<Key, Comparator>::Iterator iter(&list);
iter.Seek(i);
// Compare against model iterator
@@ -107,7 +126,7 @@ TEST(SkipTest, InsertAndLookup) {
// Backward iteration test
{
SkipList<Key, Comparator>::Iterator iter(&list);
SkipListTest<Key, Comparator>::Iterator iter(&list);
iter.SeekToLast();
// Compare against model iterator
@@ -250,7 +269,7 @@ class ConcurrentTest {
// Note that generation 0 is never inserted, so it is ok if
// <*,0,*> is missing.
ASSERT_TRUE((gen(pos) == 0) ||
(gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))
(gen(pos) > initial_state.Get(key(pos)))
) << "key: " << key(pos)
<< "; gen: " << gen(pos)
<< "; initgen: "
@@ -313,18 +332,16 @@ class TestState {
state_cv_(&mu_) {}
void Wait(ReaderState s) {
mu_.Lock();
MutexLock lock(&mu_);
while (state_ != s) {
state_cv_.Wait();
}
mu_.Unlock();
}
void Change(ReaderState s) {
mu_.Lock();
MutexLock lock(&mu_);
state_ = s;
state_cv_.Signal();
mu_.Unlock();
}
private:
@@ -371,6 +388,211 @@ TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
static void
RunSequentialInsert(
const int NumKeys,
bool AcquireLock,
bool ReverseInsert,
bool SequentialInsertModeEnabled )
{
const int loopCount = 5; // repeat the whole process this many times and average the time spent
std::vector<uint64_t> timeSpent;
port::Mutex mutex;
Env* env = Env::Default();
fprintf( stderr,
"Sequentially inserting %d keys in %s order,\n"
" seqential insert mode is initially %sabled,\n"
" %sacquiring a lock for each insert (averaging over %d runs)\n",
NumKeys, ReverseInsert ? "reverse" : "forward",
SequentialInsertModeEnabled ? "en" : "dis",
AcquireLock ? "" : "not ", loopCount );
int k;
for ( k = 0; k < loopCount; ++k ) {
int j;
Arena arena;
Comparator cmp;
SkipListTest<Key, Comparator> list( cmp, &arena );
// initially the SkipList should be in sequential mode
ASSERT_TRUE( list.InSequentialInsertMode() );
// were we instructed to disable sequential insert mode?
if ( !SequentialInsertModeEnabled ) {
list.DisableSequentialInsertMode();
ASSERT_TRUE( !list.InSequentialInsertMode() );
}
uint64_t start = env->NowMicros();
for ( j = 0; j < NumKeys; ++j ) {
Key key = ReverseInsert ? NumKeys - 1 - j : j;
if ( AcquireLock ) mutex.Lock();
list.Insert( key );
if ( AcquireLock ) mutex.Unlock();
}
uint64_t stop = env->NowMicros();
timeSpent.push_back( stop - start );
//fprintf( stderr, " Time for run %d: %llu\n", k, timeSpent[k] );
// if SequentialInsertModeEnabled is true, the SkipList should still be
// in sequential mode iff ReverseInsert is false
if ( SequentialInsertModeEnabled ) {
ASSERT_TRUE( list.InSequentialInsertMode() != ReverseInsert );
}
else {
ASSERT_TRUE( !list.InSequentialInsertMode() );
}
// ensure the SkipLlist is properly sorted
if ( AcquireLock ) mutex.Lock();
ASSERT_TRUE( list.Valid() );
if ( AcquireLock ) mutex.Unlock();
// ensure the SkipList contains all the keys we inserted
for ( j = 0; j < NumKeys; ++j ) {
ASSERT_TRUE( list.Contains( j ) );
}
}
// throw out the low and high times and average the rest
uint64_t totalTime, lowTime, highTime;
totalTime = lowTime = highTime = timeSpent[0];
for ( k = 1; k < loopCount; ++k ) {
uint64_t currentTime = timeSpent[k];
totalTime += currentTime;
if ( lowTime > currentTime ) lowTime = currentTime;
if ( highTime < currentTime ) highTime = currentTime;
}
totalTime -= (lowTime + highTime);
uint64_t averageTime = (totalTime / (loopCount - 2));
double timePerKey = (double)averageTime / (double)NumKeys;
fprintf( stderr, " Average insertion time: %" PRIu64 " (%f/key)\n", averageTime, timePerKey );
}
TEST(SkipTest, SequentialInsert_NoLock_ForwardInsert)
{
int numKeys = 100000;
bool acquireLock = false;
bool reverseInsert = false;
bool sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
sequentialInsertModeEnabled = false;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
}
TEST(SkipTest, SequentialInsert_Lock_ForwardInsert)
{
int numKeys = 100000;
bool acquireLock = true;
bool reverseInsert = false;
bool sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
sequentialInsertModeEnabled = false;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
}
TEST(SkipTest, SequentialInsert_NoLock_ReverseInsert)
{
int numKeys = 100000;
bool acquireLock = false;
bool reverseInsert = true;
bool sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
}
TEST(SkipTest, SequentialInsert_Lock_ReverseInsert)
{
int numKeys = 100000;
bool acquireLock = true;
bool reverseInsert = true;
bool sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
}
TEST(SkipTest, SequentialInsert_IncreasingNumberOfInserts)
{
// test with increasing numbers of keys, with sequential-insert mode both
// enabled and disabled; we're looking to see if per-key insertion times
// trend upward as the number of keys increases
int numKeys = 10000;
bool acquireLock = false;
bool reverseInsert = false;
bool sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
sequentialInsertModeEnabled = false;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
numKeys = 100000;
sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
sequentialInsertModeEnabled = false;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
numKeys = 1000000;
sequentialInsertModeEnabled = true;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
sequentialInsertModeEnabled = false;
RunSequentialInsert( numKeys, acquireLock, reverseInsert, sequentialInsertModeEnabled );
}
TEST(SkipTest, SequentialInsert_MixedInsertionModes)
{
// start inserting sequentially, then switch to non-sequential inserts,
// ensuring all works as intended
int j, numSequentialKeys = 100000, numNonSequentialKeys = 100000;
int totalNumKeys = numSequentialKeys + numNonSequentialKeys;
Arena arena;
Comparator cmp;
SkipListTest<Key, Comparator> list( cmp, &arena );
// initially the SkipList should be in sequential mode
ASSERT_TRUE( list.InSequentialInsertMode() );
// start inserting at key=1; when we insert 0 below, the list should switch
// out of sequential insert mode
for ( j = 1; j < numSequentialKeys; ++j ) {
list.Insert( j );
}
// the SkipList should still be in sequential mode
ASSERT_TRUE( list.InSequentialInsertMode() );
ASSERT_TRUE( list.Valid() );
list.Insert( 0 );
ASSERT_TRUE( !list.InSequentialInsertMode() );
ASSERT_TRUE( list.Valid() );
// now insert the remaining keys in non-sequential order (they're not
// random, but that doesn't matter here; just ensure we switch to
// non-sequential mode and that all continues to work)
for ( j = 0; j < numNonSequentialKeys; j += 2 ) {
int key = totalNumKeys - j - 1;
list.Insert( key );
}
for ( j = 0; j < numNonSequentialKeys; j += 2 ) {
int key = numSequentialKeys + j;
list.Insert( key );
}
ASSERT_TRUE( !list.InSequentialInsertMode() );
ASSERT_TRUE( list.Valid() );
// ensure the SkipList contains all the keys we inserted
for ( j = 0; j < totalNumKeys; ++j ) {
ASSERT_TRUE( list.Contains( j ) );
}
}
} // namespace leveldb
int main(int argc, char** argv) {
-1
View File
@@ -5,7 +5,6 @@
#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
#include "db/dbformat.h"
#include "leveldb/db.h"
namespace leveldb {
+134 -25
View File
@@ -5,22 +5,26 @@
#include "db/table_cache.h"
#include "db/filename.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
#include "db/version_edit.h"
#include "leveldb/env.h"
#include "leveldb/table.h"
#include "util/coding.h"
#include "leveldb/perf_count.h"
namespace leveldb {
struct TableAndFile {
RandomAccessFile* file;
Table* table;
};
static void DeleteEntry(const Slice& key, void* value) {
TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
delete tf->table;
delete tf->file;
delete tf;
if (0==dec_and_fetch(&tf->user_count))
{
if (NULL!=tf->doublecache)
tf->doublecache->SubFileSize(tf->table->GetFileSize());
delete tf->table;
delete tf->file;
delete tf;
} // if
}
static void UnrefEntry(void* arg1, void* arg2) {
@@ -31,37 +35,38 @@ static void UnrefEntry(void* arg1, void* arg2) {
TableCache::TableCache(const std::string& dbname,
const Options* options,
int entries)
Cache * file_cache,
DoubleCache & doublecache)
: env_(options->env),
dbname_(dbname),
options_(options),
cache_(NewLRUCache(entries)) {
cache_(file_cache),
doublecache_(doublecache)
{
}
TableCache::~TableCache() {
delete cache_;
}
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
Cache::Handle** handle) {
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, int level,
Cache::Handle** handle, bool is_compaction,
bool for_iterator) {
Status s;
char buf[sizeof(file_number)];
EncodeFixed64(buf, file_number);
Slice key(buf, sizeof(buf));
*handle = cache_->Lookup(key);
if (*handle == NULL) {
std::string fname = TableFileName(dbname_, file_number);
std::string fname = TableFileName(*options_, file_number, level);
RandomAccessFile* file = NULL;
Table* table = NULL;
s = env_->NewRandomAccessFile(fname, &file);
if (!s.ok()) {
std::string old_fname = SSTTableFileName(dbname_, file_number);
if (env_->NewRandomAccessFile(old_fname, &file).ok()) {
s = Status::OK();
}
}
if (s.ok()) {
s = Table::Open(*options_, file, file_size, &table);
// Riak: support opportunity to manage Linux page cache
if (is_compaction)
file->SetForCompaction(file_size);
}
if (!s.ok()) {
@@ -73,22 +78,74 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
TableAndFile* tf = new TableAndFile;
tf->file = file;
tf->table = table;
*handle = cache_->Insert(key, tf, 1, &DeleteEntry);
tf->doublecache = &doublecache_;
tf->file_number = file_number;
tf->level = level;
*handle = cache_->Insert(key, tf, table->TableObjectSize(), &DeleteEntry);
gPerfCounters->Inc(ePerfTableOpened);
doublecache_.AddFileSize(table->GetFileSize());
// temporary hardcoding to match number of levels defined as
// overlapped in version_set.cc
if (level<config::kNumOverlapLevels)
cache_->Addref(*handle);
}
}
else
{
Table *table = reinterpret_cast<TableAndFile*>(cache_->Value(*handle))->table;
// this is NOT first access, see if bloom filter can load now
if (!for_iterator && table->ReadFilter())
{
// TableAndFile now going to be present in two cache entries
// 1. retrieve old entry within file cache
TableAndFile* tf = reinterpret_cast<TableAndFile*>(cache_->Value(*handle));
inc_and_fetch(&tf->user_count);
// 2. must clean file size, do not want double count
if (NULL!=tf->doublecache)
tf->doublecache->SubFileSize(tf->table->GetFileSize());
// 3. release current reference (and possible special overlap reference)
cache_->Release(*handle);
if (tf->level<config::kNumOverlapLevels)
cache_->Release(*handle);
// 4. create second table cache entry using TableObjectSize that now includes
// bloom filter size
*handle = cache_->Insert(key, tf, table->TableObjectSize(), &DeleteEntry);
// 5. set double reference if an overlapped file (prevents from being flushed)
if (level<config::kNumOverlapLevels)
cache_->Addref(*handle);
} // if
// for Linux, let fadvise start precaching
if (is_compaction)
{
RandomAccessFile *file = reinterpret_cast<TableAndFile*>(cache_->Value(*handle))->file;
file->SetForCompaction(file_size);
} // if
gPerfCounters->Inc(ePerfTableCached);
} // else
return s;
}
Iterator* TableCache::NewIterator(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
int level,
Table** tableptr) {
if (tableptr != NULL) {
*tableptr = NULL;
}
Cache::Handle* handle = NULL;
Status s = FindTable(file_number, file_size, &handle);
Status s = FindTable(file_number, file_size, level, &handle, options.IsCompaction(), true);
if (!s.ok()) {
return NewErrorIterator(s);
}
@@ -105,11 +162,13 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
Status TableCache::Get(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
int level,
const Slice& k,
void* arg,
void (*saver)(void*, const Slice&, const Slice&)) {
bool (*saver)(void*, const Slice&, const Slice&)) {
Cache::Handle* handle = NULL;
Status s = FindTable(file_number, file_size, &handle);
Status s = FindTable(file_number, file_size, level, &handle);
if (s.ok()) {
Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
s = t->InternalGet(options, k, arg, saver);
@@ -118,10 +177,60 @@ Status TableCache::Get(const ReadOptions& options,
return s;
}
void TableCache::Evict(uint64_t file_number) {
void TableCache::Evict(uint64_t file_number, bool is_overlapped) {
char buf[sizeof(file_number)];
EncodeFixed64(buf, file_number);
// overlapped files have extra reference to prevent their purge,
// release that reference now
if (is_overlapped)
{
Cache::Handle *handle;
// the Lookup call adds a reference too, back out both
handle=cache_->Lookup(Slice(buf, sizeof(buf)));
// with multiple background threads, file might already be
// evicted
if (NULL!=handle)
{
cache_->Release(handle); // release for Lookup() call just made
cache_->Release(handle); // release for extra reference
} // if
} // if
cache_->Erase(Slice(buf, sizeof(buf)));
}
/**
* Riak specific routine to return table statistic ONLY if table metadata
* already within cache ... otherwise return 0.
*/
uint64_t
TableCache::GetStatisticValue(
uint64_t file_number,
unsigned Index)
{
uint64_t ret_val;
char buf[sizeof(file_number)];
Cache::Handle *handle;
ret_val=0;
EncodeFixed64(buf, file_number);
Slice key(buf, sizeof(buf));
handle = cache_->Lookup(key);
if (NULL != handle)
{
TableAndFile * tf;
tf=reinterpret_cast<TableAndFile*>(cache_->Value(handle));
ret_val=tf->table->GetSstCounters().Value(Index);
cache_->Release(handle);
} // if
return(ret_val);
} // TableCache::GetStatisticValue
} // namespace leveldb
+54 -7
View File
@@ -13,6 +13,7 @@
#include "leveldb/cache.h"
#include "leveldb/table.h"
#include "port/port.h"
#include "util/cache2.h"
namespace leveldb {
@@ -20,8 +21,10 @@ class Env;
class TableCache {
public:
TableCache(const std::string& dbname, const Options* options, int entries);
~TableCache();
// clean up note: file_cache is redundant to GetFileCache available from doublecache
TableCache(const std::string& dbname, const Options* options, Cache * file_cache,
DoubleCache & doublecache);
virtual ~TableCache();
// Return an iterator for the specified file number (the corresponding
// file length must be exactly "file_size" bytes). If "tableptr" is
@@ -33,6 +36,7 @@ class TableCache {
Iterator* NewIterator(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
int level,
Table** tableptr = NULL);
// If a seek to internal key "k" in specified file finds an entry,
@@ -40,22 +44,65 @@ class TableCache {
Status Get(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
int level,
const Slice& k,
void* arg,
void (*handle_result)(void*, const Slice&, const Slice&));
bool (*handle_result)(void*, const Slice&, const Slice&));
// Evict any entry for the specified file number
void Evict(uint64_t file_number);
void Evict(uint64_t file_number, bool is_overlapped);
private:
// Riak specific: return table statistic ONLY if table in cache, otherwise zero
uint64_t GetStatisticValue(uint64_t file_number, unsigned Index);
// access for testing tools, not for public access
Status TEST_FindTable(uint64_t file_number, uint64_t file_size, int level, Cache::Handle** handle)
{return( FindTable(file_number, file_size, level, handle));};
Cache* TEST_GetInternalCache() {return(cache_);};
void Release(Cache::Handle * handle) {cache_->Release(handle);};
// routine called if Options::cache_object_warming is true.
// Writes list of all file names currently in file cache to disk.
Status SaveOpenFileList();
// routine called if Options::cache_object_warming is true.
// Reads file created by SaveOpenFileList() and attempts to open
// every file.
Status PreloadTableCache();
// was private, now protected to allow easy unit test overrides
protected:
Env* const env_;
const std::string dbname_;
const Options* options_;
Cache* cache_;
Cache * cache_;
DoubleCache & doublecache_;
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
// virtual to enable unit test overrides
virtual Status FindTable(uint64_t file_number, uint64_t file_size, int level,
Cache::Handle**, bool is_compaction=false,
bool for_iterator=false);
};
struct TableAndFile {
RandomAccessFile* file;
Table* table;
DoubleCache * doublecache;
uint64_t file_number; // saved for cache object warming
int level; // saved for cache object warming
volatile uint32_t user_count;
TableAndFile()
: file(NULL), table(NULL), doublecache(NULL),
file_number(0), level(0), user_count(1)
{};
};
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
+50 -18
View File
@@ -9,20 +9,6 @@
namespace leveldb {
// Tag numbers for serialized VersionEdit. These numbers are written to
// disk and should not be changed.
enum Tag {
kComparator = 1,
kLogNumber = 2,
kNextFileNumber = 3,
kLastSequence = 4,
kCompactPointer = 5,
kDeletedFile = 6,
kNewFile = 7,
// 8 was used for large value refs
kPrevLogNumber = 9
};
void VersionEdit::Clear() {
comparator_.clear();
log_number_ = 0;
@@ -34,11 +20,21 @@ void VersionEdit::Clear() {
has_prev_log_number_ = false;
has_next_file_number_ = false;
has_last_sequence_ = false;
has_f1_files_ = false;
has_f2_files_ = false;
deleted_files_.clear();
new_files_.clear();
}
void VersionEdit::EncodeTo(std::string* dst) const {
/**
* EncodeTo serializes the VersionEdit object
* to the "dst" string parameter. "format2" flag
* indicates whether serialization should use original
* Google format for file objects (false) or Basho's updated
* file2 format for expiry enabled file objects (true)
*/
void VersionEdit::EncodeTo(std::string* dst, bool format2) const {
if (has_comparator_) {
PutVarint32(dst, kComparator);
PutLengthPrefixedSlice(dst, comparator_);
@@ -76,12 +72,21 @@ void VersionEdit::EncodeTo(std::string* dst) const {
for (size_t i = 0; i < new_files_.size(); i++) {
const FileMetaData& f = new_files_[i].second;
PutVarint32(dst, kNewFile);
if (format2)
PutVarint32(dst, kNewFile2);
else
PutVarint32(dst, kNewFile);
PutVarint32(dst, new_files_[i].first); // level
PutVarint64(dst, f.number);
PutVarint64(dst, f.file_size);
PutLengthPrefixedSlice(dst, f.smallest.Encode());
PutLengthPrefixedSlice(dst, f.largest.Encode());
if (format2)
{
PutVarint64(dst, f.exp_write_low);
PutVarint64(dst, f.exp_write_high);
PutVarint64(dst, f.exp_explicit_high);
}
}
}
@@ -98,7 +103,7 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
static bool GetLevel(Slice* input, int* level) {
uint32_t v;
if (GetVarint32(input, &v) &&
v < config::kNumLevels) {
v < (unsigned)config::kNumLevels) {
*level = v;
return true;
} else {
@@ -185,13 +190,34 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
GetVarint64(&input, &f.number) &&
GetVarint64(&input, &f.file_size) &&
GetInternalKey(&input, &f.smallest) &&
GetInternalKey(&input, &f.largest)) {
GetInternalKey(&input, &f.largest))
{
has_f1_files_ = true;
f.level=level;
new_files_.push_back(std::make_pair(level, f));
} else {
msg = "new-file entry";
}
break;
case kNewFile2:
if (GetLevel(&input, &level) &&
GetVarint64(&input, &f.number) &&
GetVarint64(&input, &f.file_size) &&
GetInternalKey(&input, &f.smallest) &&
GetInternalKey(&input, &f.largest) &&
GetVarint64(&input, &f.exp_write_low) &&
GetVarint64(&input, &f.exp_write_high) &&
GetVarint64(&input, &f.exp_explicit_high))
{
has_f2_files_ = true;
f.level=level;
new_files_.push_back(std::make_pair(level, f));
} else {
msg = "new-file2 entry";
}
break;
default:
msg = "unknown tag";
break;
@@ -258,6 +284,12 @@ std::string VersionEdit::DebugString() const {
r.append(f.smallest.DebugString());
r.append(" .. ");
r.append(f.largest.DebugString());
r.append(" ");
AppendNumberTo(&r, f.exp_write_low);
r.append(" ");
AppendNumberTo(&r, f.exp_write_high);
r.append(" ");
AppendNumberTo(&r, f.exp_explicit_high);
}
r.append("\n}\n");
return r;
+79 -7
View File
@@ -16,15 +16,41 @@ class VersionSet;
struct FileMetaData {
int refs;
int allowed_seeks; // Seeks allowed until compaction
// int allowed_seeks; // Seeks allowed until compaction
uint64_t number;
uint64_t file_size; // File size in bytes
uint64_t num_entries; // count of values in .sst file, only valid during table build
InternalKey smallest; // Smallest internal key served by table
InternalKey largest; // Largest internal key served by table
int level;
ExpiryTimeMicros exp_write_low; // oldest write time in file:
// 0 - non-expiry keys exist too
// ULLONG_MAX - no write time expiry & no plain keys
ExpiryTimeMicros exp_write_high; // most recent write time in file
ExpiryTimeMicros exp_explicit_high; // most recent/furthest into future explicit expiry
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
FileMetaData()
: refs(0), /*allowed_seeks(1 << 30),*/ file_size(0),
num_entries(0), level(-1), exp_write_low(0), exp_write_high(0), exp_explicit_high(0)
{ }
};
class FileMetaDataPtrCompare
{
protected:
const Comparator * comparator_;
public:
explicit FileMetaDataPtrCompare(const Comparator * Comparer)
: comparator_(Comparer) {};
bool operator() (const FileMetaData * file1, const FileMetaData * file2) const
{
return(comparator_->Compare(file1->smallest.user_key(), file2->smallest.user_key()) < 0);
}
}; // class FileMetaDataPtrCompare
class VersionEdit {
public:
VersionEdit() { Clear(); }
@@ -59,6 +85,7 @@ class VersionEdit {
// Add the specified file at the specified number.
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file
#if 0
void AddFile(int level, uint64_t file,
uint64_t file_size,
const InternalKey& smallest,
@@ -68,6 +95,27 @@ class VersionEdit {
f.file_size = file_size;
f.smallest = smallest;
f.largest = largest;
f.level = level;
new_files_.push_back(std::make_pair(level, f));
}
#endif
void AddFile2(int level, uint64_t file,
uint64_t file_size,
const InternalKey& smallest,
const InternalKey& largest,
uint64_t exp_write_low,
uint64_t exp_write_high,
uint64_t exp_explicit_high) {
FileMetaData f;
f.number = file;
f.file_size = file_size;
f.smallest = smallest;
f.largest = largest;
f.level = level;
f.exp_write_low = exp_write_low;
f.exp_write_high = exp_write_high;
f.exp_explicit_high = exp_explicit_high;
new_files_.push_back(std::make_pair(level, f));
}
@@ -75,16 +123,37 @@ class VersionEdit {
void DeleteFile(int level, uint64_t file) {
deleted_files_.insert(std::make_pair(level, file));
}
size_t DeletedFileCount() const {return(deleted_files_.size());};
void EncodeTo(std::string* dst) const;
void EncodeTo(std::string* dst, bool format2=true) const;
Status DecodeFrom(const Slice& src);
// unit test access to validate file entries' format types
bool HasF1Files() const {return(has_f1_files_);};
bool HasF2Files() const {return(has_f2_files_);};
std::string DebugString() const;
// Tag numbers for serialized VersionEdit. These numbers are written to
// disk and should not be changed.
enum Tag {
kComparator = 1,
kLogNumber = 2,
kNextFileNumber = 3,
kLastSequence = 4,
kCompactPointer = 5,
kDeletedFile = 6,
kNewFile = 7,
// 8 was used for large value refs
kPrevLogNumber = 9,
kFileCacheObject = 10,
kNewFile2 = 11 // expiry capable file
};
private:
friend class VersionSet;
typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
USED_BY_NESTED_FRIEND2(typedef std::set< std::pair<int, uint64_t> > DeletedFileSet)
std::string comparator_;
uint64_t log_number_;
@@ -96,10 +165,13 @@ class VersionEdit {
bool has_prev_log_number_;
bool has_next_file_number_;
bool has_last_sequence_;
// following should be mutually exclusive, but tested independently to be sure
bool has_f1_files_; // manifest uses format 1 (for unit tests)
bool has_f2_files_; // manifest uses format 2 (for unit tests)
std::vector< std::pair<int, InternalKey> > compact_pointers_;
DeletedFileSet deleted_files_;
std::vector< std::pair<int, FileMetaData> > new_files_;
USED_BY_NESTED_FRIEND2(std::vector< std::pair<int, InternalKey> > compact_pointers_)
USED_BY_NESTED_FRIEND(DeletedFileSet deleted_files_)
USED_BY_NESTED_FRIEND2(std::vector< std::pair<int, FileMetaData> > new_files_)
};
} // namespace leveldb
+39 -7
View File
@@ -7,14 +7,22 @@
namespace leveldb {
static void TestEncodeDecode(const VersionEdit& edit) {
static void TestEncodeDecode(
const VersionEdit& edit,
bool format2=false) {
std::string encoded, encoded2;
edit.EncodeTo(&encoded);
edit.EncodeTo(&encoded,format2);
VersionEdit parsed;
Status s = parsed.DecodeFrom(encoded);
ASSERT_TRUE(s.ok()) << s.ToString();
parsed.EncodeTo(&encoded2);
parsed.EncodeTo(&encoded2,format2);
ASSERT_EQ(encoded, encoded2);
if (parsed.HasF1Files() || parsed.HasF2Files())
{
ASSERT_EQ(parsed.HasF1Files(), !format2);
ASSERT_EQ(parsed.HasF2Files(), format2);
} // if
}
class VersionEditTest { };
@@ -25,11 +33,12 @@ TEST(VersionEditTest, EncodeDecode) {
VersionEdit edit;
for (int i = 0; i < 4; i++) {
TestEncodeDecode(edit);
edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
InternalKey("foo", kBig + 500 + i, kTypeValue),
InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
edit.AddFile2(3, kBig + 300 + i, kBig + 400 + i,
InternalKey("foo", 0, kBig + 500 + i, kTypeValue),
InternalKey("zoo", 0, kBig + 600 + i, kTypeDeletion),
0,0,0);
edit.DeleteFile(4, kBig + 700 + i);
edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
edit.SetCompactPointer(i, InternalKey("x", 0, kBig + 900 + i, kTypeValue));
}
edit.SetComparatorName("foo");
@@ -39,6 +48,29 @@ TEST(VersionEditTest, EncodeDecode) {
TestEncodeDecode(edit);
}
TEST(VersionEditTest, EncodeDecodeExpiry) {
static const uint64_t kBig = 1ull << 25;
VersionEdit edit;
for (int i = 0; i < 4; i++) {
TestEncodeDecode(edit, false); // only testing for s.ok()
edit.AddFile2(3, kBig + 300 + i, kBig + 400 + i,
InternalKey("foo", 700+i, kBig + 500 + i, kTypeValueExplicitExpiry),
InternalKey("zoo", 800+i, kBig + 600 + i, kTypeDeletion),
10203040,
123456789,
987654321);
edit.DeleteFile(4, kBig + 700 + i);
edit.SetCompactPointer(i, InternalKey("x", 0, kBig + 900 + i, kTypeValue));
}
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
edit.SetNextFile(kBig + 200);
edit.SetLastSequence(kBig + 1000);
TestEncodeDecode(edit, true);
}
} // namespace leveldb
int main(int argc, char** argv) {
Loaded 100 of 291 files, more files were not shown because too many files have changed in this diff. Show more