Pull blockchain changes into berkeleydb branch

This commit is contained in:
Thomas Winget 2015-03-17 19:52:53 -04:00
commit 8e3347f310
No known key found for this signature in database
GPG key ID: 58131A160789E630
72 changed files with 17746 additions and 110 deletions

View file

@ -65,6 +65,21 @@ option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost inst
set_property(GLOBAL PROPERTY USE_FOLDERS ON) set_property(GLOBAL PROPERTY USE_FOLDERS ON)
enable_testing() enable_testing()
# Check whether we're on a 32-bit or 64-bit system
if(CMAKE_SIZEOF_VOID_P EQUAL "8")
set(DEFAULT_BUILD_64 ON)
else()
set(DEFAULT_BUILD_64 OFF)
endif()
option(BUILD_64 "Build for 64-bit? 'OFF' builds for 32-bit." ${DEFAULT_BUILD_64})
if(BUILD_64)
set(ARCH_WIDTH "64")
else()
set(ARCH_WIDTH "32")
endif()
message(STATUS "Building for a ${ARCH_WIDTH}-bit system")
# Check if we're on FreeBSD so we can exclude the local miniupnpc (it should be installed from ports instead) # Check if we're on FreeBSD so we can exclude the local miniupnpc (it should be installed from ports instead)
# CMAKE_SYSTEM_NAME checks are commonly known, but specifically taken from libsdl's CMakeLists # CMAKE_SYSTEM_NAME checks are commonly known, but specifically taken from libsdl's CMakeLists
if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*") if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*")
@ -96,16 +111,16 @@ endif()
option(STATIC "Link libraries statically" ${DEFAULT_STATIC}) option(STATIC "Link libraries statically" ${DEFAULT_STATIC})
if(MINGW) if(MINGW)
get_filename_component(msys2_install_path "[HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\MSYS2 64bit;InstallLocation]" ABSOLUTE) get_filename_component(msys2_install_path "[HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\MSYS2 ${ARCH_WIDTH}bit;InstallLocation]" ABSOLUTE)
set(CMAKE_INCLUDE_PATH "${msys2_install_path}/mingw64/include") set(CMAKE_INCLUDE_PATH "${msys2_install_path}/mingw${ARCH_WIDTH}/include")
# This is necessary because otherwise CMake will make Boost libraries -lfoo # This is necessary because otherwise CMake will make Boost libraries -lfoo
# rather than a full path. Unfortunately, this makes the shared libraries get # rather than a full path. Unfortunately, this makes the shared libraries get
# linked due to a bug in CMake which misses putting -static flags around the # linked due to a bug in CMake which misses putting -static flags around the
# -lfoo arguments. # -lfoo arguments.
list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES
"${msys2_install_path}/mingw64/lib") "${msys2_install_path}/mingw${ARCH_WIDTH}/lib")
list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES
"${msys2_install_path}/mingw64/lib") "${msys2_install_path}/mingw${ARCH_WIDTH}/lib")
endif() endif()
if(STATIC) if(STATIC)

View file

@ -99,12 +99,3 @@ else()
endif() endif()
add_subdirectory(db_drivers) add_subdirectory(db_drivers)
set(LMDB_STATIC ${LMDB_STATIC} PARENT_SCOPE)
set(LMDB_INCLUDE ${LMDB_INCLUDE} PARENT_SCOPE)
set(LMDB_LIBRARY ${LMDB_LIBRARY} PARENT_SCOPE)
set(LMDB_LIBRARY_DIRS ${LMDB_LIBRARY_DIRS} PARENT_SCOPE)
set(BDB_STATIC ${BDB_STATIC} PARENT_SCOPE)
set(BDB_INCLUDE ${BDB_INCLUDE} PARENT_SCOPE)
set(BDB_LIBRARY ${BDB_LIBRARY} PARENT_SCOPE)
set(BDB_LIBRARY_DIRS ${BDB_LIBRARY_DIRS} PARENT_SCOPE)

View file

@ -26,27 +26,13 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
find_package(LMDB) # We aren't even going to check the system for an installed LMDB driver, as it is too
# critical a consensus component to rely on dynamically linked libraries
message(STATUS "Using ${ARCH_WIDTH}-bit LMDB from source tree")
add_subdirectory(liblmdb${ARCH_WIDTH})
set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb${ARCH_WIDTH}" CACHE STRING "LMDB Include path")
if(NOT LMDB_LIBRARIES OR STATIC) set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name")
add_subdirectory(liblmdb)
message(STATUS "lmdb not found, building from src tree")
set(LMDB_STATIC true PARENT_SCOPE)
set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb" PARENT_SCOPE)
set(LMDB_LIBRARY "lmdb" PARENT_SCOPE)
else()
message(STATUS "Found liblmdb include (lmdb.h) in ${LMDB_INCLUDE_DIR}")
if(LMDB_LIBRARIES)
message(STATUS "Found liblmdb shared library")
set(LMDB_STATIC false PARENT_SCOPE)
set(LMDB_INCLUDE ${LMDB_INCLUDE_DIR} PARENT_SCOPE)
set(LMDB_LIBRARY ${LMDB_LIBRARIES} PARENT_SCOPE)
set(LMDB_LIBRARY_DIRS "" PARENT_SCOPE)
else()
die("Found liblmdb includes, but could not find liblmdb library. Please make sure you have installed liblmdb and liblmdb-dev or the equivalent")
endif()
endif()
find_package(BerkeleyDB) find_package(BerkeleyDB)
@ -54,17 +40,17 @@ if(NOT BERKELEY_DB_LIBRARIES OR STATIC)
add_subdirectory(libdb) add_subdirectory(libdb)
message(STATUS "BerkeleyDB not found, building from src tree") message(STATUS "BerkeleyDB not found, building from src tree")
set(BDB_STATIC true PARENT_SCOPE) set(BDB_STATIC true CACHE BOOL "BDB Static flag")
set(BDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/libdb" PARENT_SCOPE) set(BDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/libdb" CACHE STRING "BDB include path")
set(BDB_LIBRARY "db" PARENT_SCOPE) set(BDB_LIBRARY "db" CACHE STRING "BDB library name")
else() else()
message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}") message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
if(BERKELEY_DB_LIBRARIES) if(BERKELEY_DB_LIBRARIES)
message(STATUS "Found BerkeleyDB shared library") message(STATUS "Found BerkeleyDB shared library")
set(BDB_STATIC false PARENT_SCOPE) set(BDB_STATIC false CACHE BOOL "BDB Static flag")
set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} PARENT_SCOPE) set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} PARENT_SCOPE) set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
set(BDB_LIBRARY_DIRS "" PARENT_SCOPE) set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
else() else()
die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent") die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
endif() endif()

View file

@ -0,0 +1,20 @@
mtest
mtest[23456]
testdb
mdb_copy
mdb_stat
*.[ao]
*.so
*.exe
*[~#]
*.bak
*.orig
*.rej
*.gcov
*.gcda
*.gcno
core
core.*
valgrind.*
man/
html/

98
external/db_drivers/liblmdb32/Makefile vendored Normal file
View file

@ -0,0 +1,98 @@
# Makefile for liblmdb (Lightning memory-mapped database library).
########################################################################
# Configuration. The compiler options must enable threaded compilation.
#
# Preprocessor macros (for CPPFLAGS) of interest...
# Note that the defaults should already be correct for most
# platforms; you should not need to change any of these.
# Read their descriptions in mdb.c if you do:
#
# - MDB_USE_POSIX_SEM
# - MDB_DSYNC
# - MDB_FDATASYNC
# - MDB_USE_PWRITEV
#
# There may be other macros in mdb.c of interest. You should
# read mdb.c before changing any of them.
#
CC = gcc
W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
THREADS = -pthread
OPT = -O2 -g
CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS)
LDLIBS =
SOLIBS =
prefix = /usr/local
XCFLAGS = -DVL32
########################################################################
IHDRS = lmdb.h
ILIBS = liblmdb.a liblmdb.so
IPROGS = mdb_stat mdb_copy mdb_dump mdb_load
IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
all: $(ILIBS) $(PROGS)
install: $(ILIBS) $(IPROGS) $(IHDRS)
for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done
for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done
for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done
for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done
clean:
rm -rf $(PROGS) *.[ao] *.so *~ testdb
test: all
mkdir testdb
./mtest && ./mdb_stat testdb
liblmdb.a: mdb.o midl.o
ar rs $@ mdb.o midl.o
liblmdb.so: mdb.o midl.o
# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS)
mdb_stat: mdb_stat.o liblmdb.a
mdb_copy: mdb_copy.o liblmdb.a
mdb_dump: mdb_dump.o liblmdb.a
mdb_load: mdb_load.o liblmdb.a
mtest: mtest.o liblmdb.a
mtest2: mtest2.o liblmdb.a
mtest3: mtest3.o liblmdb.a
mtest4: mtest4.o liblmdb.a
mtest5: mtest5.o liblmdb.a
mtest6: mtest6.o liblmdb.a
mdb.o: mdb.c lmdb.h midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c
midl.o: midl.c midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c
%: %.o
$(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@
%.o: %.c lmdb.h
$(CC) $(CFLAGS) $(CPPFLAGS) -c $<
COV_FLAGS=-fprofile-arcs -ftest-coverage
COV_OBJS=xmdb.o xmidl.o
coverage: xmtest
for i in mtest*.c [0-9]*.c; do j=`basename \$$i .c`; $(MAKE) $$j.o; \
gcc -o x$$j $$j.o $(COV_OBJS) -pthread $(COV_FLAGS); \
rm -rf testdb; mkdir testdb; ./x$$j; done
gcov xmdb.c
gcov xmidl.c
xmtest: mtest.o xmdb.o xmidl.o
gcc -o xmtest mtest.o xmdb.o xmidl.o -pthread $(COV_FLAGS)
xmdb.o: mdb.c lmdb.h midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c mdb.c -o $@
xmidl.o: midl.c midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c midl.c -o $@

1553
external/db_drivers/liblmdb32/lmdb.h vendored Normal file

File diff suppressed because it is too large Load diff

9572
external/db_drivers/liblmdb32/mdb.c vendored Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,82 @@
/* mdb_copy.c - memory-mapped database backup tool */
/*
* Copyright 2012 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#ifdef _WIN32
#include <windows.h>
#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE)
#else
#define MDB_STDOUT 1
#endif
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include "lmdb.h"
static void
sighandle(int sig)
{
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
const char *progname = argv[0], *act;
unsigned flags = MDB_RDONLY;
unsigned cpflags = 0;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDB_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDB_CP_COMPACT;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("%s\n", MDB_VERSION_STRING);
exit(0);
} else
argc = 0;
}
if (argc<2 || argc>3) {
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
exit(EXIT_FAILURE);
}
#ifdef SIGPIPE
signal(SIGPIPE, sighandle);
#endif
#ifdef SIGHUP
signal(SIGHUP, sighandle);
#endif
signal(SIGINT, sighandle);
signal(SIGTERM, sighandle);
act = "opening environment";
rc = mdb_env_create(&env);
if (rc == MDB_SUCCESS) {
rc = mdb_env_open(env, argv[1], flags, 0664);
}
if (rc == MDB_SUCCESS) {
act = "copying";
if (argc == 2)
rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags);
else
rc = mdb_env_copy2(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n",
progname, act, rc, mdb_strerror(rc));
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

190
external/db_drivers/liblmdb32/midl.h vendored Normal file
View file

@ -0,0 +1,190 @@
/** @file midl.h
* @brief LMDB ID List header file.
*
* This file was originally part of back-bdb but has been
* modified for use in libmdb. Most of the macros defined
* in this file are unused, just left over from the original.
*
* This file is only used internally in libmdb and its definitions
* are not exposed publicly.
*/
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2000-2014 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#ifndef _MDB_MIDL_H_
#define _MDB_MIDL_H_
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/** @defgroup internal LMDB Internals
* @{
*/
/** @defgroup idls ID List Management
* @{
*/
/** A generic unsigned ID number. These were entryIDs in back-bdb.
* Preferably it should have the same size as a pointer.
*/
typedef size_t MDB_ID;
/** An IDL is an ID List, a sorted array of IDs. The first
* element of the array is a counter for how many actual
* IDs are in the list. In the original back-bdb code, IDLs are
* sorted in ascending order. For libmdb IDLs are sorted in
* descending order.
*/
typedef MDB_ID *MDB_IDL;
/* IDL sizes - likely should be even bigger
* limiting factors: sizeof(ID), thread stack size
*/
#ifdef VL32
#define MDB_IDL_LOGN 10 /* DB_SIZE is 2^10, UM_SIZE is 2^11 */
#else
#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
#endif
#define MDB_IDL_DB_SIZE (1<<MDB_IDL_LOGN)
#define MDB_IDL_UM_SIZE (1<<(MDB_IDL_LOGN+1))
#define MDB_IDL_DB_MAX (MDB_IDL_DB_SIZE-1)
#define MDB_IDL_UM_MAX (MDB_IDL_UM_SIZE-1)
#define MDB_IDL_SIZEOF(ids) (((ids)[0]+1) * sizeof(MDB_ID))
#define MDB_IDL_IS_ZERO(ids) ( (ids)[0] == 0 )
#define MDB_IDL_CPY( dst, src ) (memcpy( dst, src, MDB_IDL_SIZEOF( src ) ))
#define MDB_IDL_FIRST( ids ) ( (ids)[1] )
#define MDB_IDL_LAST( ids ) ( (ids)[(ids)[0]] )
/** Current max length of an #mdb_midl_alloc()ed IDL */
#define MDB_IDL_ALLOCLEN( ids ) ( (ids)[-1] )
/** Append ID to IDL. The IDL must be big enough. */
#define mdb_midl_xappend(idl, id) do { \
MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \
xidl[xlen] = (id); \
} while (0)
/** Search for an ID in an IDL.
* @param[in] ids The IDL to search.
* @param[in] id The ID to search for.
* @return The index of the first ID greater than or equal to \b id.
*/
unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id );
/** Allocate an IDL.
* Allocates memory for an IDL of the given size.
* @return IDL on success, NULL on failure.
*/
MDB_IDL mdb_midl_alloc(int num);
/** Free an IDL.
* @param[in] ids The IDL to free.
*/
void mdb_midl_free(MDB_IDL ids);
/** Shrink an IDL.
* Return the IDL to the default size if it has grown larger.
* @param[in,out] idp Address of the IDL to shrink.
* @return 0 on no change, non-zero if shrunk.
*/
int mdb_midl_shrink(MDB_IDL *idp);
/** Make room for num additional elements in an IDL.
* @param[in,out] idp Address of the IDL.
* @param[in] num Number of elements to make room for.
* @return 0 on success, ENOMEM on failure.
*/
int mdb_midl_need(MDB_IDL *idp, unsigned num);
/** Append an ID onto an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] id The ID to append.
* @return 0 on success, ENOMEM if the IDL is too large.
*/
int mdb_midl_append( MDB_IDL *idp, MDB_ID id );
/** Append an IDL onto an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] app The IDL to append.
* @return 0 on success, ENOMEM if the IDL is too large.
*/
int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app );
/** Append an ID range onto an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] id The lowest ID to append.
* @param[in] n Number of IDs to append.
* @return 0 on success, ENOMEM if the IDL is too large.
*/
int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n );
/** Merge an IDL onto an IDL. The destination IDL must be big enough.
* @param[in] idl The IDL to merge into.
* @param[in] merge The IDL to merge.
*/
void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge );
/** Sort an IDL.
* @param[in,out] ids The IDL to sort.
*/
void mdb_midl_sort( MDB_IDL ids );
/** An ID2 is an ID/pointer pair.
*/
typedef struct MDB_ID2 {
MDB_ID mid; /**< The ID */
void *mptr; /**< The pointer */
} MDB_ID2;
/** An ID2L is an ID2 List, a sorted array of ID2s.
* The first element's \b mid member is a count of how many actual
* elements are in the array. The \b mptr member of the first element is unused.
* The array is sorted in ascending order by \b mid.
*/
typedef MDB_ID2 *MDB_ID2L;
/** Search for an ID in an ID2L.
* @param[in] ids The ID2L to search.
* @param[in] id The ID to search for.
* @return The index of the first ID2 whose \b mid member is greater than or equal to \b id.
*/
unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id );
/** Insert an ID2 into a ID2L.
* @param[in,out] ids The ID2L to insert into.
* @param[in] id The ID2 to insert.
* @return 0 on success, -1 if the ID was already present in the ID2L.
*/
int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
/** Append an ID2 into a ID2L.
* @param[in,out] ids The ID2L to append into.
* @param[in] id The ID2 to append.
* @return 0 on success, -2 if the ID2L is too big.
*/
int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
/** @} */
/** @} */
#ifdef __cplusplus
}
#endif
#endif /* _MDB_MIDL_H_ */

173
external/db_drivers/liblmdb32/mtest.c vendored Normal file
View file

@ -0,0 +1,173 @@
/* mtest.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor, *cur2;
MDB_cursor_op op;
int count;
int *values;
char sval[32] = "";
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_open(env, "./testdb", 0 /* MDB_FIXEDMAP |MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, NULL, 0, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
data.mv_data = sval;
}
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last/prev\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cur2));
for (i=0; i<50; i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
E(mdb_del(txn, dbi, &key, NULL));
}
printf("Restarting cursor in txn\n");
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cur2);
E(mdb_txn_commit(txn));
printf("Restarting cursor outside txn\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}

112
external/db_drivers/liblmdb64/CHANGES vendored Normal file
View file

@ -0,0 +1,112 @@
LMDB 0.9 Change Log
LMDB 0.9.14 Release (2014/09/20)
Fix to support 64K page size (ITS#7713)
Fix to persist decreased as well as increased mapsizes (ITS#7789)
Fix cursor bug when deleting last node of a DUPSORT key
Fix mdb_env_info to return FIXEDMAP address
Fix ambiguous error code from writing to closed DBI (ITS#7825)
Fix mdb_copy copying past end of file (ITS#7886)
Fix cursor bugs from page_merge/rebalance
Fix to dirty fewer pages in deletes (mdb_page_loose())
Fix mdb_dbi_open creating subDBs (ITS#7917)
Fix mdb_cursor_get(_DUP) with single value (ITS#7913)
Fix Windows compat issues in mtests (ITS#7879)
Add compacting variant of mdb_copy
Add BigEndian integer key compare code
Add mdb_dump/mdb_load utilities
LMDB 0.9.13 Release (2014/06/18)
Fix mdb_page_alloc unlimited overflow page search
Documentation
Re-fix MDB_CURRENT doc (ITS#7793)
Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc
LMDB 0.9.12 Release (2014/06/13)
Fix MDB_GET_BOTH regression (ITS#7875,#7681)
Fix MDB_MULTIPLE writing multiple keys (ITS#7834)
Fix mdb_rebalance (ITS#7829)
Fix mdb_page_split (ITS#7815)
Fix md_entries count (ITS#7861,#7828,#7793)
Fix MDB_CURRENT (ITS#7793)
Fix possible crash on Windows DLL detach
Misc code cleanup
Documentation
mdb_cursor_put: cursor moves on error (ITS#7771)
LMDB 0.9.11 Release (2014/01/15)
Add mdb_env_set_assert() (ITS#7775)
Fix: invalidate txn on page allocation errors (ITS#7377)
Fix xcursor tracking in mdb_cursor_del0() (ITS#7771)
Fix corruption from deletes (ITS#7756)
Fix Windows/MSVC build issues
Raise safe limit of max MDB_MAXKEYSIZE
Misc code cleanup
Documentation
Remove spurious note about non-overlapping flags (ITS#7665)
LMDB 0.9.10 Release (2013/11/12)
Add MDB_NOMEMINIT option
Fix mdb_page_split() again (ITS#7589)
Fix MDB_NORDAHEAD definition (ITS#7734)
Fix mdb_cursor_del() positioning (ITS#7733)
Partial fix for larger page sizes (ITS#7713)
Fix Windows64/MSVC build issues
LMDB 0.9.9 Release (2013/10/24)
Add mdb_env_get_fd()
Add MDB_NORDAHEAD option
Add MDB_NOLOCK option
Avoid wasting space in mdb_page_split() (ITS#7589)
Fix mdb_page_merge() cursor fixup (ITS#7722)
Fix mdb_cursor_del() on last delete (ITS#7718)
Fix adding WRITEMAP on existing env (ITS#7715)
Fix nested txns (ITS#7515)
Fix mdb_env_copy() O_DIRECT bug (ITS#7682)
Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681)
Fix mdb_rebalance() cursor fixup (ITS#7701)
Misc code cleanup
Documentation
Note that by default, readers need write access
LMDB 0.9.8 Release (2013/09/09)
Allow mdb_env_set_mapsize() on an open environment
Fix mdb_dbi_flags() (ITS#7672)
Fix mdb_page_unspill() in nested txns
Fix mdb_cursor_get(CURRENT|NEXT) after a delete
Fix mdb_cursor_get(DUP) to always return key (ITS#7671)
Fix mdb_cursor_del() to always advance to next item (ITS#7670)
Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681)
Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682)
Tweak mdb_page_spill() to be less aggressive
Documentation
Update caveats since mdb_reader_check() added in 0.9.7
LMDB 0.9.7 Release (2013/08/17)
Don't leave stale lockfile on failed RDONLY open (ITS#7664)
Fix mdb_page_split() ref beyond cursor depth
Fix read txn data race (ITS#7635)
Fix mdb_rebalance (ITS#7536, #7538)
Fix mdb_drop() (ITS#7561)
Misc DEBUG macro fixes
Add MDB_NOTLS envflag
Add mdb_env_copyfd()
Add mdb_txn_env() (ITS#7660)
Add mdb_dbi_flags() (ITS#7661)
Add mdb_env_get_maxkeysize()
Add mdb_env_reader_list()/mdb_env_reader_check()
Add mdb_page_spill/unspill, remove hard txn size limit
Use shorter names for semaphores (ITS#7615)
Build
Fix install target (ITS#7656)
Documentation
Misc updates for cursors, DB handles, data lifetime
LMDB 0.9.6 Release (2013/02/25)
Many fixes/enhancements
LMDB 0.9.5 Release (2012/11/30)
Renamed from libmdb to liblmdb
Many fixes/enhancements

View file

@ -1,4 +1,5 @@
# Copyright (c) 2014, The Monero Project # Copyright (c) 2014-2015, The Monero Project
#
# All rights reserved. # All rights reserved.
# #
# Redistribution and use in source and binary forms, with or without modification, are # Redistribution and use in source and binary forms, with or without modification, are
@ -25,33 +26,14 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
MESSAGE(STATUS "Looking for liblmdb") set (lmdb_sources
mdb.c
midl.c)
FIND_PATH(LMDB_INCLUDE_DIR include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
NAMES lmdb.h
PATH_SUFFIXES include/ include/lmdb/
PATHS "${PROJECT_SOURCE_DIR}"
${LMDB_ROOT}
$ENV{LMDB_ROOT}
/usr/local/
/usr/
)
if(STATIC) add_library(lmdb
if(MINGW) ${lmdb_sources})
find_library(LMDB_LIBRARIES liblmdb.dll.a) target_link_libraries(lmdb
else() LINK_PRIVATE
find_library(LMDB_LIBRARIES liblmdb.a) ${CMAKE_THREAD_LIBS_INIT})
endif()
else()
find_library(LMDB_LIBRARIES lmdb)
endif()
IF(LMDB_INCLUDE_DIR)
MESSAGE(STATUS "Found liblmdb include (lmdb.h) in ${LMDB_INCLUDE_DIR}")
IF(LMDB_LIBRARIES)
MESSAGE(STATUS "Found liblmdb library")
set(LMDB_INCLUDE ${LMDB_INCLUDE_DIR})
set(LMDB_LIBRARY ${LMDB_LIBRARIES})
ENDIF()
ENDIF()

20
external/db_drivers/liblmdb64/COPYRIGHT vendored Normal file
View file

@ -0,0 +1,20 @@
Copyright 2011-2014 Howard Chu, Symas Corp.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted only as authorized by the OpenLDAP
Public License.
A copy of this license is available in the file LICENSE in the
top-level directory of the distribution or, alternatively, at
<http://www.OpenLDAP.org/license.html>.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Individual files and/or contributed packages may be copyright by
other parties and/or subject to additional restrictions.
This work also contains materials derived from public sources.
Additional information about OpenLDAP can be obtained at
<http://www.openldap.org/>.

1631
external/db_drivers/liblmdb64/Doxyfile vendored Normal file

File diff suppressed because it is too large Load diff

47
external/db_drivers/liblmdb64/LICENSE vendored Normal file
View file

@ -0,0 +1,47 @@
The OpenLDAP Public License
Version 2.8, 17 August 2003
Redistribution and use of this software and associated documentation
("Software"), with or without modification, are permitted provided
that the following conditions are met:
1. Redistributions in source form must retain copyright statements
and notices,
2. Redistributions in binary form must reproduce applicable copyright
statements and notices, this list of conditions, and the following
disclaimer in the documentation and/or other materials provided
with the distribution, and
3. Redistributions must contain a verbatim copy of this document.
The OpenLDAP Foundation may revise this license from time to time.
Each revision is distinguished by a version number. You may use
this Software under terms of this license revision or under the
terms of any subsequent revision of the license.
THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
The names of the authors and copyright holders must not be used in
advertising or otherwise to promote the sale, use or other dealing
in this Software without specific, written prior permission. Title
to copyright in this Software shall at all times remain with copyright
holders.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
California, USA. All Rights Reserved. Permission to copy and
distribute verbatim copies of this document is granted.

View file

@ -0,0 +1,54 @@
.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_copy \- LMDB environment copy tool
.SH SYNOPSIS
.B mdb_copy
[\c
.BR \-V ]
[\c
.BR \-c ]
[\c
.BR \-n ]
.B srcpath
[\c
.BR dstpath ]
.SH DESCRIPTION
The
.B mdb_copy
utility copies an LMDB environment. The environment can
be copied regardless of whether it is currently in use.
No lockfile is created, since it gets recreated at need.
If
.I dstpath
is specified it must be the path of an empty directory
for storing the backup. Otherwise, the backup will be
written to stdout.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-c
Compact while copying. Only current data pages will be copied; freed
or unused pages will be omitted from the copy. This option will
slow down the backup process as it is more CPU-intensive.
.TP
.BR \-n
Open LDMB environment(s) which do not use subdirectories.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH CAVEATS
This utility can trigger significant file size growth if run
in parallel with write transactions, because pages which they
free during copying cannot be reused until the copy is done.
.SH "SEE ALSO"
.BR mdb_stat (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

View file

@ -0,0 +1,75 @@
.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_dump \- LMDB environment export tool
.SH SYNOPSIS
.B mdb_dump
.BR \ envpath
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-l ]
[\c
.BR \-n ]
[\c
.BR \-p ]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.SH DESCRIPTION
The
.B mdb_dump
utility reads a database and writes its contents to the
standard output using a portable flat-text format
understood by the
.BR mdb_load (1)
utility.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-f \ file
Write to the specified file instead of to the standard output.
.TP
.BR \-l
List the databases stored in the environment. Just the
names will be listed, no data will be output.
.TP
.BR \-n
Dump an LMDB database which does not use subdirectories.
.TP
.BR \-p
If characters in either the key or data items are printing characters (as
defined by isprint(3)), output them directly. This option permits users to
use standard text editors and tools to modify the contents of databases.
Note: different systems may have different notions about what characters
are considered printing characters, and databases dumped in this manner may
be less portable to external systems.
.TP
.BR \-a
Dump all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Dump a specific subdatabase. If no database is specified, only the main database is dumped.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
Dumping and reloading databases that use user-defined comparison functions
will result in new databases that use the default comparison functions.
\fBIn this case it is quite likely that the reloaded database will be
damaged beyond repair permitting neither record storage nor retrieval.\fP
The only available workaround is to modify the source for the
.BR mdb_load (1)
utility to load the database using the correct comparison functions.
.SH "SEE ALSO"
.BR mdb_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

317
external/db_drivers/liblmdb64/mdb_dump.c vendored Normal file
View file

@ -0,0 +1,317 @@
/* mdb_dump.c - memory-mapped database dump tool */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include <signal.h>
#include "lmdb.h"
#ifdef _WIN32
#define Z "I"
#else
#define Z "z"
#endif
#define PRINT 1
static int mode;
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {
{ MDB_REVERSEKEY, "reversekey" },
{ MDB_DUPSORT, "dupsort" },
{ MDB_INTEGERKEY, "integerkey" },
{ MDB_DUPFIXED, "dupfixed" },
{ MDB_INTEGERDUP, "integerdup" },
{ MDB_REVERSEDUP, "reversedup" },
{ 0, NULL }
};
static volatile sig_atomic_t gotsig;
static void dumpsig( int sig )
{
gotsig=1;
}
static const char hexc[] = "0123456789abcdef";
static void hex(unsigned char c)
{
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
}
static void text(MDB_val *v)
{
unsigned char *c, *end;
putchar(' ');
c = v->mv_data;
end = c + v->mv_size;
while (c < end) {
if (isprint(*c)) {
putchar(*c);
} else {
putchar('\\');
hex(*c);
}
c++;
}
putchar('\n');
}
static void byte(MDB_val *v)
{
unsigned char *c, *end;
putchar(' ');
c = v->mv_data;
end = c + v->mv_size;
while (c < end) {
hex(*c++);
}
putchar('\n');
}
/* Dump in BDB-compatible format */
static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
{
MDB_cursor *mc;
MDB_stat ms;
MDB_val key, data;
MDB_envinfo info;
unsigned int flags;
int rc, i;
rc = mdb_dbi_flags(txn, dbi, &flags);
if (rc) return rc;
rc = mdb_stat(txn, dbi, &ms);
if (rc) return rc;
rc = mdb_env_info(mdb_txn_env(txn), &info);
if (rc) return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" Z "u\n", info.me_mapsize);
if (info.me_mapaddr)
printf("mapaddr=%p\n", info.me_mapaddr);
printf("maxreaders=%u\n", info.me_maxreaders);
if (flags & MDB_DUPSORT)
printf("duplicates=1\n");
for (i=0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
printf("db_pagesize=%d\n", ms.ms_psize);
printf("HEADER=END\n");
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) return rc;
while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) {
if (gotsig) {
rc = EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
} else {
byte(&key);
byte(&data);
}
}
printf("DATA=END\n");
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
return rc;
}
static void usage(char *prog)
{
fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envflags = 0, list = 0;
if (argc < 2) {
usage(prog);
}
/* -a: dump main DB and all subDBs
* -s: dump only the named subDB
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -V: print version and exit
* (default) dump only the main DB
*/
while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'l':
list = 1;
/*FALLTHROUGH*/;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'f':
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'p':
mode |= PRINT;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#ifdef SIGPIPE
signal(SIGPIPE, dumpsig);
#endif
#ifdef SIGHUP
signal(SIGHUP, dumpsig);
#endif
signal(SIGINT, dumpsig);
signal(SIGTERM, dumpsig);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 2);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
int count = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
count++;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS) {
if (list) {
printf("%s\n", str);
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdb_close(env, db2);
}
free(str);
if (rc) continue;
}
mdb_cursor_close(cursor);
if (!count) {
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname);
rc = MDB_NOTFOUND;
} else if (rc == MDB_NOTFOUND) {
rc = MDB_SUCCESS;
}
} else {
rc = dumpit(txn, dbi, subname);
}
if (rc && rc != MDB_NOTFOUND)
fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc));
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

View file

@ -0,0 +1,77 @@
.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_load \- LMDB environment import tool
.SH SYNOPSIS
.B mdb_load
.BR \ envpath
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-n ]
[\c
.BI \-s \ subdb\fR]
[\c
.BR \-N ]
[\c
.BR \-T ]
.SH DESCRIPTION
The
.B mdb_load
utility reads from the standard input and loads it into the
LMDB environment
.BR envpath .
The input to
.B mdb_load
must be in the output format specified by the
.BR mdb_dump (1)
utility or as specified by the
.B -T
option below.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-f \ file
Read from the specified file instead of from the standard input.
.TP
.BR \-n
Load an LMDB database which does not use subdirectories.
.TP
.BR \-s \ subdb
Load a specific subdatabase. If no database is specified, data is loaded into the main database.
.TP
.BR \-N
Don't overwrite existing records when loading into an already existing database; just skip them.
.TP
.BR \-T
Load data from simple text files. The input must be paired lines of text, where the first
line of the pair is the key item, and the second line of the pair is its corresponding
data item.
A simple escape mechanism, where newline and backslash (\\) characters are special, is
applied to the text input. Newline characters are interpreted as record separators.
Backslash characters in the text will be interpreted in one of two ways: If the backslash
character precedes another backslash character, the pair will be interpreted as a literal
backslash. If the backslash character precedes any other character, the two characters
following the backslash will be interpreted as a hexadecimal specification of a single
character; for example, \\0a is a newline character in the ASCII character set.
For this reason, any backslash or newline characters that naturally occur in the text
input must be escaped to avoid misinterpretation by
.BR mdb_load .
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdb_dump (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

452
external/db_drivers/liblmdb64/mdb_load.c vendored Normal file
View file

@ -0,0 +1,452 @@
/* mdb_load.c - memory-mapped database load tool */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "lmdb.h"
#define PRINT 1
#define NOHDR 2
static int mode;
static char *subname = NULL;
static size_t lineno;
static int version;
static int flags;
static char *prog;
static int Eof;
static MDB_envinfo info;
static MDB_val kbuf, dbuf;
#ifdef _WIN32
#define Z "I"
#else
#define Z "z"
#endif
#define STRLENOF(s) (sizeof(s)-1)
typedef struct flagbit {
int bit;
char *name;
int len;
} flagbit;
#define S(s) s, STRLENOF(s)
flagbit dbflags[] = {
{ MDB_REVERSEKEY, S("reversekey") },
{ MDB_DUPSORT, S("dupsort") },
{ MDB_INTEGERKEY, S("integerkey") },
{ MDB_DUPFIXED, S("dupfixed") },
{ MDB_INTEGERDUP, S("integerdup") },
{ MDB_REVERSEDUP, S("reversedup") },
{ 0, NULL, 0 }
};
static void readhdr(void)
{
char *ptr;
while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
break;
} else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
if (subname) free(subname);
subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) {
fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i=0; dbflags[i].bit; i++) {
if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
flags |= dbflags[i].bit;
break;
}
}
if (!dbflags[i].bit) {
ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
if (!ptr) {
fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
prog, lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.mv_data);
}
}
}
}
}
static void badend(void)
{
fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
prog, lineno);
}
static int unhex(unsigned char *c2)
{
int x, c;
x = *c2++ & 0x4f;
if (x & 0x40)
x -= 55;
c = x << 4;
x = *c2 & 0x4f;
if (x & 0x40)
x -= 55;
c |= x;
return c;
}
static int readline(MDB_val *out, MDB_val *buf)
{
unsigned char *c1, *c2, *end;
size_t len;
int c;
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
badend:
Eof = 1;
badend();
return EOF;
}
if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
return EOF;
goto badend;
}
}
if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
Eof = 1;
return EOF;
}
lineno++;
c1 = buf->mv_data;
len = strlen((char *)c1);
/* Is buffer too short? */
while (c1[len-1] != '\n') {
buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
if (!buf->mv_data) {
Eof = 1;
fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->mv_data;
c1 += buf->mv_size;
if (fgets((char *)c1, buf->mv_size, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->mv_size *= 2;
len = strlen((char *)c1);
}
c1 = c2 = buf->mv_data;
len = strlen((char *)c1);
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
if (*c2 == '\\') {
if (c2[1] == '\\') {
c1++; c2 += 2;
} else {
if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = unhex(++c2);
c2 += 2;
}
} else {
c1++; c2++;
}
}
} else {
/* odd length not allowed */
if (len & 1) {
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = unhex(c2);
c2 += 2;
}
}
c2 = out->mv_data = buf->mv_data;
out->mv_size = c1 - c2;
return 0;
}
static void usage(void)
{
fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_cursor *mc;
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
int dohdr = 0;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
break;
case 'T':
mode |= NOHDR;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
while(!Eof) {
MDB_val key, data;
int batch = 0;
flags = 0;
if (!dohdr) {
dohdr = 1;
} else if (!(mode & NOHDR))
readhdr();
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while(1) {
rc = readline(&key, &kbuf);
if (rc == EOF)
break;
if (rc)
goto txn_abort;
rc = readline(&data, &dbuf);
if (rc)
goto txn_abort;
rc = mdb_cursor_put(mc, &key, &data, putflags);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc)
goto txn_abort;
batch++;
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
}
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

View file

@ -0,0 +1,64 @@
.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_stat \- LMDB environment status tool
.SH SYNOPSIS
.B mdb_stat
.BR \ envpath
[\c
.BR \-V ]
[\c
.BR \-e ]
[\c
.BR \-f [ f [ f ]]]
[\c
.BR \-n ]
[\c
.BR \-r [ r ]]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.SH DESCRIPTION
The
.B mdb_stat
utility displays the status of an LMDB environment.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-e
Display information about the database environment.
.TP
.BR \-f
Display information about the environment freelist.
If \fB\-ff\fP is given, summarize each freelist entry.
If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
.TP
.BR \-n
Display the status of an LMDB database which does not use subdirectories.
.TP
.BR \-r
Display information about the environment reader table.
Shows the process ID, thread ID, and transaction ID for each active
reader slot. The process ID and transaction ID are in decimal, the
thread ID is in hexadecimal. The transaction ID is displayed as "-"
if the reader does not currently have a read transaction open.
If \fB\-rr\fP is given, check for stale entries in the reader
table and clear them. The reader table will be printed again
after the check is performed.
.TP
.BR \-a
Display the status of all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Display the status of a specific subdatabase.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdb_copy (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

263
external/db_drivers/liblmdb64/mdb_stat.c vendored Normal file
View file

@ -0,0 +1,263 @@
/* mdb_stat.c - memory-mapped database status tool */
/*
* Copyright 2011-2013 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "lmdb.h"
#ifdef _WIN32
#define Z "I"
#else
#define Z "z"
#endif
static void prstat(MDB_stat *ms)
{
#if 0
printf(" Page size: %u\n", ms->ms_psize);
#endif
printf(" Tree depth: %u\n", ms->ms_depth);
printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages);
printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages);
printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages);
printf(" Entries: %"Z"u\n", ms->ms_entries);
}
static void usage(char *prog)
{
fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDB_stat mst;
MDB_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 4);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (envinfo) {
(void)mdb_env_stat(env, &mst);
(void)mdb_env_info(env, &mei);
printf("Environment Info\n");
printf(" Map address: %p\n", mei.me_mapaddr);
printf(" Map size: %"Z"u\n", mei.me_mapsize);
printf(" Page size: %u\n", mst.ms_psize);
printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize);
printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1);
printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid);
printf(" Max readers: %u\n", mei.me_maxreaders);
printf(" Number of readers used: %u\n", mei.me_numreaders);
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdb_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (freinfo) {
MDB_cursor *cursor;
MDB_val key, data;
size_t pages = 0, *iptr;
printf("Freelist Status\n");
dbi = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
iptr = data.mv_data;
pages += *iptr;
if (freinfo > 1) {
char *bad = "";
size_t pg, prev;
ssize_t i, j, span = 0;
j = *iptr++;
for (i = j, prev = 1; --i >= 0; ) {
pg = iptr[i];
if (pg <= prev)
bad = " [bad sequence]";
prev = pg;
pg += span;
for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
}
printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n",
*(size_t *)key.mv_data, j, span, bad);
if (freinfo > 2) {
for (--j; j >= 0; ) {
pg = iptr[j];
for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n",
pg, span);
}
}
}
}
mdb_cursor_close(cursor);
printf(" Free pages: %"Z"u\n", pages);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS)
printf("Status of %s\n", str);
free(str);
if (rc) continue;
rc = mdb_stat(txn, db2, &mst);
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdb_close(env, db2);
}
mdb_cursor_close(cursor);
}
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

360
external/db_drivers/liblmdb64/midl.c vendored Normal file
View file

@ -0,0 +1,360 @@
/** @file midl.c
* @brief ldap bdb back-end ID List functions */
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2000-2014 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <limits.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/types.h>
#include "midl.h"
/** @defgroup internal LMDB Internals
* @{
*/
/** @defgroup idls ID List Management
* @{
*/
#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) )
unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
{
/*
* binary search of id in ids
* if found, returns position of id
* if not found, returns first position greater than id
*/
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
unsigned n = ids[0];
while( 0 < n ) {
unsigned pivot = n >> 1;
cursor = base + pivot + 1;
val = CMP( ids[cursor], id );
if( val < 0 ) {
n = pivot;
} else if ( val > 0 ) {
base = cursor;
n -= pivot + 1;
} else {
return cursor;
}
}
if( val > 0 ) {
++cursor;
}
return cursor;
}
#if 0 /* superseded by append/sort */
int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
{
unsigned x, i;
x = mdb_midl_search( ids, id );
assert( x > 0 );
if( x < 1 ) {
/* internal error */
return -2;
}
if ( x <= ids[0] && ids[x] == id ) {
/* duplicate */
assert(0);
return -1;
}
if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
/* no room */
--ids[0];
return -2;
} else {
/* insert id */
for (i=ids[0]; i>x; i--)
ids[i] = ids[i-1];
ids[x] = id;
}
return 0;
}
#endif
MDB_IDL mdb_midl_alloc(int num)
{
MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID));
if (ids) {
*ids++ = num;
*ids = 0;
}
return ids;
}
void mdb_midl_free(MDB_IDL ids)
{
if (ids)
free(ids-1);
}
int mdb_midl_shrink( MDB_IDL *idp )
{
MDB_IDL ids = *idp;
if (*(--ids) > MDB_IDL_UM_MAX &&
(ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID))))
{
*ids++ = MDB_IDL_UM_MAX;
*idp = ids;
return 1;
}
return 0;
}
static int mdb_midl_grow( MDB_IDL *idp, int num )
{
MDB_IDL idn = *idp-1;
/* grow it */
idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID));
if (!idn)
return ENOMEM;
*idn++ += num;
*idp = idn;
return 0;
}
int mdb_midl_need( MDB_IDL *idp, unsigned num )
{
MDB_IDL ids = *idp;
num += ids[0];
if (num > ids[-1]) {
num = (num + num/4 + (256 + 2)) & -256;
if (!(ids = realloc(ids-1, num * sizeof(MDB_ID))))
return ENOMEM;
*ids++ = num - 2;
*idp = ids;
}
return 0;
}
int mdb_midl_append( MDB_IDL *idp, MDB_ID id )
{
MDB_IDL ids = *idp;
/* Too big? */
if (ids[0] >= ids[-1]) {
if (mdb_midl_grow(idp, MDB_IDL_UM_MAX))
return ENOMEM;
ids = *idp;
}
ids[0]++;
ids[ids[0]] = id;
return 0;
}
int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app )
{
MDB_IDL ids = *idp;
/* Too big? */
if (ids[0] + app[0] >= ids[-1]) {
if (mdb_midl_grow(idp, app[0]))
return ENOMEM;
ids = *idp;
}
memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID));
ids[0] += app[0];
return 0;
}
int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n )
{
MDB_ID *ids = *idp, len = ids[0];
/* Too big? */
if (len + n > ids[-1]) {
if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX))
return ENOMEM;
ids = *idp;
}
ids[0] = len + n;
ids += len;
while (n)
ids[n--] = id++;
return 0;
}
void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge )
{
MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k;
idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */
old_id = idl[j];
while (i) {
merge_id = merge[i--];
for (; old_id < merge_id; old_id = idl[--j])
idl[k--] = old_id;
idl[k--] = merge_id;
}
idl[0] = total;
}
/* Quicksort + Insertion sort for small arrays */
#define SMALL 8
#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; }
void
mdb_midl_sort( MDB_IDL ids )
{
/* Max possible depth of int-indexed tree * 2 items/level */
int istack[sizeof(int)*CHAR_BIT * 2];
int i,j,k,l,ir,jstack;
MDB_ID a, itmp;
ir = (int)ids[0];
l = 1;
jstack = 0;
for(;;) {
if (ir - l < SMALL) { /* Insertion sort */
for (j=l+1;j<=ir;j++) {
a = ids[j];
for (i=j-1;i>=1;i--) {
if (ids[i] >= a) break;
ids[i+1] = ids[i];
}
ids[i+1] = a;
}
if (jstack == 0) break;
ir = istack[jstack--];
l = istack[jstack--];
} else {
k = (l + ir) >> 1; /* Choose median of left, center, right */
MIDL_SWAP(ids[k], ids[l+1]);
if (ids[l] < ids[ir]) {
MIDL_SWAP(ids[l], ids[ir]);
}
if (ids[l+1] < ids[ir]) {
MIDL_SWAP(ids[l+1], ids[ir]);
}
if (ids[l] < ids[l+1]) {
MIDL_SWAP(ids[l], ids[l+1]);
}
i = l+1;
j = ir;
a = ids[l+1];
for(;;) {
do i++; while(ids[i] > a);
do j--; while(ids[j] < a);
if (j < i) break;
MIDL_SWAP(ids[i],ids[j]);
}
ids[l+1] = ids[j];
ids[j] = a;
jstack += 2;
if (ir-i+1 >= j-l) {
istack[jstack] = ir;
istack[jstack-1] = i;
ir = j-1;
} else {
istack[jstack] = j-1;
istack[jstack-1] = l;
l = i;
}
}
}
}
unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id )
{
/*
* binary search of id in ids
* if found, returns position of id
* if not found, returns first position greater than id
*/
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
unsigned n = (unsigned)ids[0].mid;
while( 0 < n ) {
unsigned pivot = n >> 1;
cursor = base + pivot + 1;
val = CMP( id, ids[cursor].mid );
if( val < 0 ) {
n = pivot;
} else if ( val > 0 ) {
base = cursor;
n -= pivot + 1;
} else {
return cursor;
}
}
if( val > 0 ) {
++cursor;
}
return cursor;
}
int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id )
{
unsigned x, i;
x = mdb_mid2l_search( ids, id->mid );
if( x < 1 ) {
/* internal error */
return -2;
}
if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
/* duplicate */
return -1;
}
if ( ids[0].mid >= MDB_IDL_UM_MAX ) {
/* too big */
return -2;
} else {
/* insert id */
ids[0].mid++;
for (i=(unsigned)ids[0].mid; i>x; i--)
ids[i] = ids[i-1];
ids[x] = *id;
}
return 0;
}
int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
{
/* Too big? */
if (ids[0].mid >= MDB_IDL_UM_MAX) {
return -2;
}
ids[0].mid++;
ids[ids[0].mid] = *id;
return 0;
}
/** @} */
/** @} */

123
external/db_drivers/liblmdb64/mtest2.c vendored Normal file
View file

@ -0,0 +1,123 @@
/* mtest2.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Just like mtest.c, but using a subDB instead of the main DB */
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}

133
external/db_drivers/liblmdb64/mtest3.c vendored Normal file
View file

@ -0,0 +1,133 @@
/* mtest3.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for sorted duplicate DBs */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}

168
external/db_drivers/liblmdb64/mtest4.c vendored Normal file
View file

@ -0,0 +1,168 @@
/* mtest4.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for sorted duplicate DBs with fixed-size keys */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[8];
char kval[sizeof(int)];
memset(sval, 0, sizeof(sval));
count = 510;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = i*5;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
strcpy(kval, "001");
for (i=0;i<count;i++) {
sprintf(sval, "%07x", values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
/* test all 3 branches of split code:
* 1: new key in lower half
* 2: new key at split point
* 3: new key in upper half
*/
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%07x", values[3]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[255]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[500]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
E(mdb_txn_commit(txn));
/* Try MDB_NEXT_MULTIPLE */
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%3)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%07x", values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}

135
external/db_drivers/liblmdb64/mtest5.c vendored Normal file
View file

@ -0,0 +1,135 @@
/* mtest5.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for sorted duplicate DBs using cursor_put */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
mdb_cursor_close(cursor);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
return 0;
}

137
external/db_drivers/liblmdb64/mtest6.c vendored Normal file
View file

@ -0,0 +1,137 @@
/* mtest6.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for DB splits and merges */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
char dkbuf[1024];
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
long kval;
char *sval;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
data.mv_size = mst.ms_psize / 4 - 30;
data.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
#endif
mdb_env_close(env);
return 0;
}

View file

@ -0,0 +1,73 @@
/* sample-bdb.txt - BerkeleyDB toy/sample
*
* Do a line-by-line comparison of this and sample-mdb.txt
*/
/*
* Copyright 2012 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <string.h>
#include <db.h>
int main(int argc,char * argv[])
{
int rc;
DB_ENV *env;
DB *dbi;
DBT key, data;
DB_TXN *txn;
DBC *cursor;
char sval[32], kval[32];
/* Note: Most error checking omitted for simplicity */
#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD)
rc = db_env_create(&env, 0);
rc = env->open(env, "./testdb", FLAGS, 0664);
rc = db_create(&dbi, env, 0);
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664);
memset(&key, 0, sizeof(DBT));
memset(&data, 0, sizeof(DBT));
key.size = sizeof(int);
key.data = sval;
data.size = sizeof(sval);
data.data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = dbi->put(dbi, txn, &key, &data, 0);
rc = txn->commit(txn, 0);
if (rc) {
fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc));
goto leave;
}
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->cursor(dbi, txn, &cursor, 0);
key.flags = DB_DBT_USERMEM;
key.data = kval;
key.ulen = sizeof(kval);
data.flags = DB_DBT_USERMEM;
data.data = sval;
data.ulen = sizeof(sval);
while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.data, (int) key.size, (char *) key.data,
data.data, (int) data.size, (char *) data.data);
}
rc = cursor->c_close(cursor);
rc = txn->abort(txn);
leave:
rc = dbi->close(dbi, 0);
rc = env->close(env, 0);
return rc;
}

View file

@ -0,0 +1,62 @@
/* sample-mdb.txt - MDB toy/sample
*
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2012 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include "lmdb.h"
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_cursor *cursor;
char sval[32];
/* Note: Most error checking omitted for simplicity */
rc = mdb_env_create(&env);
rc = mdb_env_open(env, "./testdb", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, 0, &dbi);
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdb_put(txn, dbi, &key, &data, 0);
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc));
goto leave;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_close(env, dbi);
mdb_env_close(env);
return 0;
}

22
external/db_drivers/liblmdb64/tooltag vendored Normal file
View file

@ -0,0 +1,22 @@
<tagfile>
<compound kind="page">
<name>mdb_copy_1</name>
<title>mdb_copy - environment copy tool</title>
<filename>mdb_copy.1</filename>
</compound>
<compound kind="page">
<name>mdb_dump_1</name>
<title>mdb_dump - environment export tool</title>
<filename>mdb_dump.1</filename>
</compound>
<compound kind="page">
<name>mdb_load_1</name>
<title>mdb_load - environment import tool</title>
<filename>mdb_load.1</filename>
</compound>
<compound kind="page">
<name>mdb_stat_1</name>
<title>mdb_stat - environment status tool</title>
<filename>mdb_stat.1</filename>
</compound>
</tagfile>

View file

@ -1,3 +1,31 @@
# Copyright (c) 2014-2015, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required(VERSION 2.8.7) cmake_minimum_required(VERSION 2.8.7)
project(unbound C) project(unbound C)

View file

@ -35,6 +35,32 @@ set(blockchain_converter_private_headers)
bitmonero_private_headers(blockchain_converter bitmonero_private_headers(blockchain_converter
${blockchain_converter_private_headers}) ${blockchain_converter_private_headers})
set(blockchain_import_sources
blockchain_import.cpp
)
set(blockchain_import_private_headers
import.h
fake_core.h
)
bitmonero_private_headers(blockchain_import
${blockchain_import_private_headers})
set(blockchain_export_sources
blockchain_export.cpp
)
set(blockchain_export_private_headers
import.h
blockchain_export.h
)
bitmonero_private_headers(blockchain_export
${blockchain_export_private_headers})
if (BLOCKCHAIN_DB STREQUAL DB_LMDB) if (BLOCKCHAIN_DB STREQUAL DB_LMDB)
bitmonero_add_executable(blockchain_converter bitmonero_add_executable(blockchain_converter
${blockchain_converter_sources} ${blockchain_converter_sources}
@ -52,3 +78,33 @@ set_property(TARGET blockchain_converter
PROPERTY PROPERTY
OUTPUT_NAME "blockchain_converter") OUTPUT_NAME "blockchain_converter")
endif () endif ()
bitmonero_add_executable(blockchain_import
${blockchain_import_sources}
${blockchain_import_private_headers})
target_link_libraries(blockchain_import
LINK_PRIVATE
cryptonote_core
${CMAKE_THREAD_LIBS_INIT})
add_dependencies(blockchain_import
version)
set_property(TARGET blockchain_import
PROPERTY
OUTPUT_NAME "blockchain_import")
bitmonero_add_executable(blockchain_export
${blockchain_export_sources}
${blockchain_export_private_headers})
target_link_libraries(blockchain_export
LINK_PRIVATE
cryptonote_core
${CMAKE_THREAD_LIBS_INIT})
add_dependencies(blockchain_export
version)
set_property(TARGET blockchain_export
PROPERTY
OUTPUT_NAME "blockchain_export")

View file

@ -38,52 +38,179 @@
#include "cryptonote_core/blockchain.h" #include "cryptonote_core/blockchain.h"
#include "blockchain_db/lmdb/db_lmdb.h" #include "blockchain_db/lmdb/db_lmdb.h"
#include "cryptonote_core/tx_pool.h" #include "cryptonote_core/tx_pool.h"
#include "common/command_line.h"
#include "serialization/json_utils.h"
#include "include_base_utils.h"
#include "version.h"
#include <iostream> #include <iostream>
// CONFIG
static bool opt_batch = true;
static bool opt_testnet = false;
// number of blocks per batch transaction
// adjustable through command-line argument according to available RAM
static uint64_t db_batch_size = 20000;
namespace po = boost::program_options;
using namespace cryptonote; using namespace cryptonote;
using namespace epee;
struct fake_core struct fake_core
{ {
tx_memory_pool m_pool;
Blockchain dummy; Blockchain dummy;
tx_memory_pool m_pool;
blockchain_storage m_storage; blockchain_storage m_storage;
#if !defined(BLOCKCHAIN_DB)
fake_core(const boost::filesystem::path &path) : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) // for multi_db_runtime:
fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(&dummy), m_storage(m_pool)
#else
// for multi_db_compile:
fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(dummy), m_storage(&m_pool)
#endif
{ {
m_pool.init(path.string()); m_pool.init(path.string());
m_storage.init(path.string(), false); m_storage.init(path.string(), use_testnet);
} }
}; };
int main(int argc, char* argv[]) int main(int argc, char* argv[])
{ {
std::string dir = tools::get_default_data_dir(); uint64_t height = 0;
boost::filesystem::path default_data_path {dir}; uint64_t start_block = 0;
if (argc >= 2 && !strcmp(argv[1], "--testnet")) { uint64_t end_block = 0;
default_data_path /= "testnet"; uint64_t num_blocks = 0;
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", LOG_LEVEL_0};
const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size};
const command_line::arg_descriptor<bool> arg_testnet_on = {
"testnet"
, "Run on testnet."
, opt_testnet
};
const command_line::arg_descriptor<uint64_t> arg_block_number =
{"block-number", "Number of blocks (default: use entire source blockchain)",
0};
command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_batch_size);
command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_block_number);
command_line::add_arg(desc_cmd_only, command_line::arg_help);
const command_line::arg_descriptor<bool> arg_batch = {"batch",
"Batch transactions for faster import", true};
// call add_options() directly for these arguments since command_line helpers
// support only boolean switch, not boolean argument
desc_cmd_sett.add_options()
(arg_batch.name, make_semantic(arg_batch), arg_batch.description)
;
po::options_description desc_options("Allowed options");
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
po::store(po::parse_command_line(argc, argv, desc_options), vm);
po::notify(vm);
return true;
});
if (!r)
return 1;
int log_level = command_line::get_arg(vm, arg_log_level);
opt_batch = command_line::get_arg(vm, arg_batch);
db_batch_size = command_line::get_arg(vm, arg_batch_size);
if (command_line::get_arg(vm, command_line::arg_help))
{
std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL;
std::cout << desc_options << std::endl;
return 1;
} }
fake_core c(default_data_path); if (! opt_batch && ! vm["batch-size"].defaulted())
{
std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL;
return 1;
}
if (! db_batch_size)
{
std::cerr << "Error: batch-size must be > 0" << ENDL;
return 1;
}
log_space::get_set_log_detalisation_level(true, log_level);
log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
LOG_PRINT_L0("Starting...");
std::string src_folder;
opt_testnet = command_line::get_arg(vm, arg_testnet_on);
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
src_folder = command_line::get_arg(vm, data_dir_arg);
boost::filesystem::path dest_folder(src_folder);
num_blocks = command_line::get_arg(vm, arg_block_number);
if (opt_batch)
{
LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha
<< " batch size: " << db_batch_size);
}
else
{
LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha);
}
LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
fake_core c(src_folder, opt_testnet);
height = c.m_storage.get_current_blockchain_height();
if (! num_blocks || num_blocks > height)
end_block = height - 1;
else
end_block = start_block + num_blocks - 1;
BlockchainDB *blockchain; BlockchainDB *blockchain;
blockchain = new BlockchainLMDB(opt_batch);
dest_folder /= blockchain->get_db_name();
LOG_PRINT_L0("Source blockchain: " << src_folder);
LOG_PRINT_L0("Dest blockchain: " << dest_folder.string());
LOG_PRINT_L0("Opening LMDB: " << dest_folder.string());
blockchain->open(dest_folder.string());
blockchain = new BlockchainLMDB(); if (opt_batch)
blockchain->batch_start();
boost::filesystem::path db_path(default_data_path); uint64_t i = 0;
for (i = start_block; i < end_block + 1; ++i)
db_path /= blockchain->get_db_name();
blockchain->open(db_path.string());
for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i)
{ {
if (i % 10 == 0) // block: i height: i+1 end height: end_block + 1
if ((i+1) % 10 == 0)
{ {
std::cout << "\r \r" << "block " << i << "/" << height std::cout << "\r \r" << "height " << i+1 << "/" <<
<< " (" << (i+1)*100/height<< "%)" << std::flush; end_block+1 << " (" << (i+1)*100/(end_block+1)<< "%)" << std::flush;
} }
// for debugging:
// std::cout << "height " << i+1 << "/" << end_block+1
// << " ((" << i+1 << ")*100/(end_block+1))" << "%)" << ENDL;
block b = c.m_storage.get_block(i); block b = c.m_storage.get_block(i);
size_t bsize = c.m_storage.get_block_size(i); size_t bsize = c.m_storage.get_block_size(i);
difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i);
@ -94,8 +221,8 @@ int main(int argc, char* argv[])
c.m_storage.get_transactions(b.tx_hashes, txs, missed); c.m_storage.get_transactions(b.tx_hashes, txs, missed);
if (missed.size()) if (missed.size())
{ {
std::cout << std::endl; std::cout << ENDL;
std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; std::cerr << "Missed transaction(s) for block at height " << i + 1 << ", exiting" << ENDL;
delete blockchain; delete blockchain;
return 1; return 1;
} }
@ -103,17 +230,40 @@ int main(int argc, char* argv[])
try try
{ {
blockchain->add_block(b, bsize, bdiff, bcoins, txs); blockchain->add_block(b, bsize, bdiff, bcoins, txs);
if (opt_batch)
{
if ((i < end_block) && ((i + 1) % db_batch_size == 0))
{
std::cout << "\r \r";
std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL;
blockchain->batch_stop();
blockchain->batch_start();
std::cout << ENDL;
blockchain->show_stats();
}
}
} }
catch (const std::exception& e) catch (const std::exception& e)
{ {
std::cout << std::endl; std::cout << ENDL;
std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL;
delete blockchain; delete blockchain;
return 2; return 2;
} }
} }
if (opt_batch)
{
std::cout << "\r \r" << "height " << i << "/" <<
end_block+1 << " (" << (i)*100/(end_block+1)<< "%)" << std::flush;
std::cout << ENDL;
std::cout << "[- batch commit at height " << i << " -]" << ENDL;
blockchain->batch_stop();
}
std::cout << ENDL;
blockchain->show_stats();
std::cout << "Finished at height: " << i << " block: " << i-1 << ENDL;
std::cout << std::endl;
delete blockchain; delete blockchain;
return 0; return 0;
} }

View file

@ -0,0 +1,386 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <algorithm>
#include <cstdio>
#include <fstream>
#include <boost/iostreams/copy.hpp>
#include <atomic>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/filter/bzip2.hpp>
#include "common/command_line.h"
#include "version.h"
#include "blockchain_export.h"
#include "cryptonote_core/cryptonote_boost_serialization.h"
#include "import.h"
static int max_chunk = 0;
static size_t height;
namespace po = boost::program_options;
using namespace cryptonote;
using namespace epee;
bool BlockchainExport::open(const boost::filesystem::path& dir_path)
{
if (boost::filesystem::exists(dir_path))
{
if (!boost::filesystem::is_directory(dir_path))
{
LOG_PRINT_RED_L0("export directory path is a file: " << dir_path);
return false;
}
}
else
{
if (!boost::filesystem::create_directory(dir_path))
{
LOG_PRINT_RED_L0("Failed to create directory " << dir_path);
return false;
}
}
std::string file_path = (dir_path / BLOCKCHAIN_RAW).string();
m_raw_data_file = new std::ofstream();
m_raw_data_file->open(file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc);
if (m_raw_data_file->fail())
return false;
m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream);
if (m_raw_archive == NULL)
return false;
return true;
}
void BlockchainExport::flush_chunk()
{
m_output_stream->flush();
char buffer[STR_LENGTH_OF_INT + 1];
int chunk_size = (int) m_buffer.size();
if (chunk_size > BUFFER_SIZE)
{
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
}
sprintf(buffer, STR_FORMAT_OF_INT, chunk_size);
m_raw_data_file->write(buffer, STR_LENGTH_OF_INT);
if (max_chunk < chunk_size)
{
max_chunk = chunk_size;
}
long pos_before = m_raw_data_file->tellp();
std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
m_raw_data_file->flush();
long pos_after = m_raw_data_file->tellp();
long num_chars_written = pos_after - pos_before;
if ((int) num_chars_written != chunk_size)
{
LOG_PRINT_RED_L0("INTERNAL ERROR: num chars wrote NEQ buffer size. height = " << height);
}
m_buffer.clear();
delete m_raw_archive;
delete m_output_stream;
m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream);
}
void BlockchainExport::serialize_block_to_text_buffer(const block& block)
{
*m_raw_archive << block;
}
void BlockchainExport::buffer_serialize_tx(const transaction& tx)
{
*m_raw_archive << tx;
}
void BlockchainExport::buffer_write_num_txs(const std::list<transaction> txs)
{
int n = txs.size();
*m_raw_archive << n;
}
void BlockchainExport::write_block(block& block)
{
serialize_block_to_text_buffer(block);
std::list<transaction> txs;
uint64_t block_height = boost::get<txin_gen>(block.miner_tx.vin.front()).height;
// put coinbase transaction first
transaction coinbase_tx = block.miner_tx;
crypto::hash coinbase_tx_hash = get_transaction_hash(coinbase_tx);
#if SOURCE_DB == DB_MEMORY
const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash);
#else
transaction cb_tx_full = m_blockchain_storage->get_db()->get_tx(coinbase_tx_hash);
#endif
#if SOURCE_DB == DB_MEMORY
if (cb_tx_full != NULL)
{
txs.push_back(*cb_tx_full);
}
#else
// TODO: should check and abort if cb_tx_full equals null_hash?
txs.push_back(cb_tx_full);
#endif
// now add all regular transactions
BOOST_FOREACH(const auto& tx_id, block.tx_hashes)
{
#if SOURCE_DB == DB_MEMORY
const transaction* tx = m_blockchain_storage->get_tx(tx_id);
#else
transaction tx = m_blockchain_storage->get_db()->get_tx(tx_id);
#endif
#if SOURCE_DB == DB_MEMORY
if(tx == NULL)
{
if (! m_tx_pool)
throw std::runtime_error("Aborting: tx == NULL, so memory pool required to get tx, but memory pool isn't enabled");
else
{
transaction tx;
if(m_tx_pool->get_transaction(tx_id, tx))
txs.push_back(tx);
else
throw std::runtime_error("Aborting: tx not found in pool");
}
}
else
txs.push_back(*tx);
#else
txs.push_back(tx);
#endif
}
// serialize all txs to the persistant storage
buffer_write_num_txs(txs);
BOOST_FOREACH(const auto& tx, txs)
{
buffer_serialize_tx(tx);
}
// These three attributes are currently necessary for a fast import that adds blocks without verification.
bool include_extra_block_data = true;
if (include_extra_block_data)
{
#if SOURCE_DB == DB_MEMORY
size_t block_size = m_blockchain_storage->get_block_size(block_height);
#else
size_t block_size = m_blockchain_storage->get_db()->get_block_size(block_height);
#endif
#if SOURCE_DB == DB_MEMORY
difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height);
#else
difficulty_type cumulative_difficulty = m_blockchain_storage->get_db()->get_block_cumulative_difficulty(block_height);
#endif
#if SOURCE_DB == DB_MEMORY
uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height);
#else
// TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number
uint64_t coins_generated = m_blockchain_storage->get_db()->get_block_already_generated_coins(block_height);
#endif
*m_raw_archive << block_size;
*m_raw_archive << cumulative_difficulty;
*m_raw_archive << coins_generated;
}
}
bool BlockchainExport::BlockchainExport::close()
{
if (m_raw_data_file->fail())
return false;
m_raw_data_file->flush();
delete m_raw_archive;
delete m_output_stream;
delete m_raw_data_file;
return true;
}
#if SOURCE_DB == DB_MEMORY
bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height)
#else
bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height)
#endif
{
uint64_t use_block_height2 = 0;
m_blockchain_storage = _blockchain_storage;
m_tx_pool = _tx_pool;
LOG_PRINT_L0("Storing blocks raw data...");
if (!BlockchainExport::open(output_dir))
{
LOG_PRINT_RED_L0("failed to open raw file for write");
return false;
}
block b;
LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height());
LOG_PRINT_L0("requested block height: " << use_block_height);
if ((use_block_height > 0) && (use_block_height < m_blockchain_storage->get_current_blockchain_height()))
use_block_height2 = use_block_height;
else
{
use_block_height2 = m_blockchain_storage->get_current_blockchain_height();
LOG_PRINT_L0("using block height: " << use_block_height2);
}
for (height=0; height < use_block_height2; ++height)
{
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height);
m_blockchain_storage->get_block_by_hash(hash, b);
write_block(b);
if (height % NUM_BLOCKS_PER_CHUNK == 0) {
flush_chunk();
}
}
if (height % NUM_BLOCKS_PER_CHUNK != 0)
{
flush_chunk();
}
LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes");
return BlockchainExport::close();
}
int main(int argc, char* argv[])
{
uint32_t log_level = 0;
uint64_t block_height = 0;
std::string import_filename = BLOCKCHAIN_RAW;
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level};
const command_line::arg_descriptor<uint64_t> arg_block_height = {"block-number", "", block_height};
const command_line::arg_descriptor<bool> arg_testnet_on = {
"testnet"
, "Run on testnet."
, false
};
command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_block_height);
command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_only, command_line::arg_help);
po::options_description desc_options("Allowed options");
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
po::store(po::parse_command_line(argc, argv, desc_options), vm);
po::notify(vm);
return true;
});
if (! r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL;
std::cout << desc_options << std::endl;
return 1;
}
log_level = command_line::get_arg(vm, arg_log_level);
block_height = command_line::get_arg(vm, arg_block_height);
log_space::get_set_log_detalisation_level(true, log_level);
log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
LOG_PRINT_L0("Starting...");
LOG_PRINT_L0("Setting log level = " << log_level);
bool opt_testnet = command_line::get_arg(vm, arg_testnet_on);
std::string m_config_folder;
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
m_config_folder = command_line::get_arg(vm, data_dir_arg);
boost::filesystem::path output_dir {m_config_folder};
output_dir /= "export";
LOG_PRINT_L0("Export directory: " << output_dir.string());
// If we wanted to use the memory pool, we would set up a fake_core.
#if SOURCE_DB == DB_MEMORY
// blockchain_storage* core_storage = NULL;
// tx_memory_pool m_mempool(*core_storage); // is this fake anyway? just passing in NULL! so m_mempool can't be used anyway, right?
// core_storage = new blockchain_storage(&m_mempool);
blockchain_storage* core_storage = new blockchain_storage(NULL);
LOG_PRINT_L0("Initializing source blockchain (in-memory database)");
r = core_storage->init(m_config_folder, opt_testnet);
#else
// Use Blockchain instead of lower-level BlockchainDB for two reasons:
// 1. Blockchain has the init() method for easy setup
// 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
//
// cannot match blockchain_storage setup above with just one line,
// e.g.
// Blockchain* core_storage = new Blockchain(NULL);
// because unlike blockchain_storage constructor, which takes a pointer to
// tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
Blockchain* core_storage = NULL;
tx_memory_pool m_mempool(*core_storage);
core_storage = new Blockchain(m_mempool);
r = core_storage->init(m_config_folder, opt_testnet);
#endif
CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
LOG_PRINT_L0("Exporting blockchain raw data...");
BlockchainExport be;
r = be.store_blockchain_raw(core_storage, NULL, output_dir, block_height);
CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
LOG_PRINT_L0("Blockchain raw data exported OK");
}

View file

@ -0,0 +1,87 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <boost/archive/binary_oarchive.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include "cryptonote_core/cryptonote_basic.h"
#include "cryptonote_core/blockchain_storage.h"
#include "cryptonote_core/blockchain.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
// CONFIG: choose one of the three #define's
//
// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows
// the exporter to use the in-memory blockchain while the other binaries
// work with LMDB, without recompiling anything.
//
#define SOURCE_DB DB_MEMORY
// #define SOURCE_DB DB_LMDB
// to use global compile-time setting (DB_MEMORY or DB_LMDB):
// #define SOURCE_DB BLOCKCHAIN_DB
using namespace cryptonote;
class BlockchainExport
{
public:
#if SOURCE_DB == DB_MEMORY
bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp,
boost::filesystem::path& output_dir, uint64_t use_block_height=0);
#else
bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
boost::filesystem::path& output_dir, uint64_t use_block_height=0);
#endif
protected:
#if SOURCE_DB == DB_MEMORY
blockchain_storage* m_blockchain_storage;
#else
Blockchain* m_blockchain_storage;
#endif
tx_memory_pool* m_tx_pool;
typedef std::vector<char> buffer_type;
std::ofstream * m_raw_data_file;
boost::archive::binary_oarchive * m_raw_archive;
buffer_type m_buffer;
boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream;
// open export file for write
bool open(const boost::filesystem::path& dir_path);
bool close();
void write_block(block& block);
void serialize_block_to_text_buffer(const block& block);
void buffer_serialize_tx(const transaction& tx);
void buffer_write_num_txs(const std::list<transaction> txs);
void flush_chunk();
};

View file

@ -0,0 +1,751 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <atomic>
#include <cstdio>
#include <algorithm>
#include <fstream>
#include <boost/filesystem.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include "cryptonote_core/cryptonote_basic.h"
#include "cryptonote_core/cryptonote_format_utils.h"
#include "cryptonote_core/cryptonote_boost_serialization.h"
#include "serialization/json_utils.h" // dump_json()
#include "include_base_utils.h"
#include "common/command_line.h"
#include "version.h"
#include <lmdb.h> // for db flag arguments
#include "import.h"
#include "fake_core.h"
// CONFIG
static bool opt_batch = true;
static bool opt_verify = true; // use add_new_block, which does verification before calling add_block
static bool opt_resume = true;
static bool opt_testnet = true;
// number of blocks per batch transaction
// adjustable through command-line argument according to available RAM
static uint64_t db_batch_size = 20000;
namespace po = boost::program_options;
using namespace cryptonote;
using namespace epee;
int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags)
{
std::vector<std::string> db_args;
boost::split(db_args, db_arg_str, boost::is_any_of("#"));
db_engine = db_args.front();
boost::algorithm::trim(db_engine);
if (db_args.size() == 1)
{
return 0;
}
else if (db_args.size() > 2)
{
std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL;
return 1;
}
std::string db_arg_str2 = db_args[1];
boost::split(db_args, db_arg_str2, boost::is_any_of(","));
for (auto& it : db_args)
{
boost::algorithm::trim(it);
if (it.empty())
continue;
LOG_PRINT_L1("LMDB flag: " << it);
if (it == "nosync")
{
mdb_flags |= MDB_NOSYNC;
}
else if (it == "nometasync")
{
mdb_flags |= MDB_NOMETASYNC;
}
else if (it == "writemap")
{
mdb_flags |= MDB_WRITEMAP;
}
else if (it == "mapasync")
{
mdb_flags |= MDB_MAPASYNC;
}
else
{
std::cerr << "unrecognized database flag: " << it << ENDL;
return 1;
}
}
return 0;
}
int count_blocks(std::string& import_file_path)
{
boost::filesystem::path raw_file_path(import_file_path);
boost::system::error_code ec;
if (!boost::filesystem::exists(raw_file_path, ec))
{
LOG_PRINT_L0("import file not found: " << raw_file_path);
throw std::runtime_error("Aborting");
}
std::ifstream import_file;
import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
uint64_t h = 0;
if (import_file.fail())
{
LOG_PRINT_L0("import_file.open() fail");
throw std::runtime_error("Aborting");
}
LOG_PRINT_L0("Scanning blockchain from import file...");
char buffer1[STR_LENGTH_OF_INT + 1];
block b;
transaction tx;
bool quit = false;
uint64_t bytes_read = 0;
int progress_interval = 10;
while (! quit)
{
int chunk_size;
import_file.read(buffer1, STR_LENGTH_OF_INT);
if (!import_file) {
std::cout << "\r \r";
LOG_PRINT_L1("End of import file reached");
quit = true;
break;
}
h += NUM_BLOCKS_PER_CHUNK;
if (h % progress_interval == 0)
{
std::cout << "\r \r" << "block height: " << h <<
std::flush;
}
bytes_read += STR_LENGTH_OF_INT;
buffer1[STR_LENGTH_OF_INT] = '\0';
chunk_size = atoi(buffer1);
if (chunk_size > BUFFER_SIZE)
{
std::cout << "\r \r";
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
<< " height: " << h);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > 100000)
{
std::cout << "\r \r";
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << " height: "
<< h);
}
else if (chunk_size <= 0) {
std::cout << "\r \r";
LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h);
throw std::runtime_error("Aborting");
}
// skip to next expected block size value
import_file.seekg(chunk_size, std::ios_base::cur);
if (! import_file) {
std::cout << "\r \r";
LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: "
<< import_file.gcount() << " of chunk_size " << chunk_size);
throw std::runtime_error("Aborting");
}
bytes_read += chunk_size;
std::cout << "\r \r";
LOG_PRINT_L3("Total bytes scanned: " << bytes_read);
}
import_file.close();
std::cout << ENDL;
std::cout << "Done scanning import file" << ENDL;
std::cout << "Total bytes scanned: " << bytes_read << ENDL;
std::cout << "Height: " << h << ENDL;
return h;
}
template <typename FakeCore>
int import_from_file(FakeCore& simple_core, std::string& import_file_path)
{
#if !defined(BLOCKCHAIN_DB)
static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value,
"FakeCore constraint error");
#endif
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
if (std::is_same<fake_core_lmdb, FakeCore>::value)
{
// Reset stats, in case we're using newly created db, accumulating stats
// from addition of genesis block.
// This aligns internal db counts with importer counts.
simple_core.m_storage.get_db()->reset_stats();
}
#endif
boost::filesystem::path raw_file_path(import_file_path);
boost::system::error_code ec;
if (!boost::filesystem::exists(raw_file_path, ec))
{
LOG_PRINT_L0("import file not found: " << raw_file_path);
return false;
}
uint64_t source_height = count_blocks(import_file_path);
LOG_PRINT_L0("import file blockchain height: " << source_height);
std::ifstream import_file;
import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
uint64_t h = 0;
if (import_file.fail())
{
LOG_PRINT_L0("import_file.open() fail");
return false;
}
char buffer1[STR_LENGTH_OF_INT + 1];
char buffer_block[BUFFER_SIZE];
block b;
transaction tx;
int quit = 0;
uint64_t bytes_read = 0;
uint64_t start_height = 1;
if (opt_resume)
start_height = simple_core.m_storage.get_current_blockchain_height();
// Note that a new blockchain will start with a height of 1 (block number 0)
// due to genesis block being added at initialization.
// CONFIG
// TODO: can expand on this, e.g. with --block-number option
uint64_t stop_height = source_height;
// These are what we'll try to use, and they don't have to be a determination
// from source and destination blockchains, but those are the current
// defaults.
LOG_PRINT_L0("start height: " << start_height << " stop height: " <<
stop_height);
bool use_batch = false;
if (opt_batch)
{
if (simple_core.support_batch)
use_batch = true;
else
LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
}
if (use_batch)
simple_core.batch_start();
LOG_PRINT_L0("Reading blockchain from import file...");
std::cout << ENDL;
// Within the loop, we skip to start_height before we start adding.
// TODO: Not a bottleneck, but we can use what's done in count_blocks() and
// only do the chunk size reads, skipping the chunk content reads until we're
// at start_height.
while (! quit)
{
int chunk_size;
import_file.read(buffer1, STR_LENGTH_OF_INT);
if (! import_file) {
std::cout << "\r \r";
LOG_PRINT_L0("End of import file reached");
quit = 1;
break;
}
bytes_read += STR_LENGTH_OF_INT;
buffer1[STR_LENGTH_OF_INT] = '\0';
chunk_size = atoi(buffer1);
if (chunk_size > BUFFER_SIZE)
{
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > 100000)
{
LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000");
}
else if (chunk_size < 0) {
LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " < 0");
return 2;
}
import_file.read(buffer_block, chunk_size);
if (! import_file) {
LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: "
<< import_file.gcount() << " of chunk_size " << chunk_size);
return 2;
}
bytes_read += chunk_size;
LOG_PRINT_L3("Total bytes read: " << bytes_read);
if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1)
{
h += NUM_BLOCKS_PER_CHUNK;
continue;
}
if (h > stop_height)
{
LOG_PRINT_L0("Specified height reached - stopping. height: " << h << " block: " << h-1);
quit = 1;
break;
}
try
{
boost::iostreams::basic_array_source<char> device(buffer_block, chunk_size);
boost::iostreams::stream<boost::iostreams::basic_array_source<char>> s(device);
boost::archive::binary_iarchive a(s);
int display_interval = 1000;
int progress_interval = 10;
for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; chunk_ind++)
{
h++;
if (h % display_interval == 0)
{
std::cout << "\r \r";
LOG_PRINT_L0("loading block height " << h);
}
else
{
LOG_PRINT_L3("loading block height " << h);
}
try {
a >> b;
}
catch (const std::exception& e)
{
std::cout << "\r \r";
LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h);
quit = 1;
break;
}
LOG_PRINT_L2("block prev_id: " << b.prev_id << ENDL);
if (h % progress_interval == 0)
{
std::cout << "\r \r" << "block " << h-1
<< std::flush;
}
std::vector<transaction> txs;
int num_txs;
try
{
a >> num_txs;
}
catch (const std::exception& e)
{
std::cout << "\r \r";
LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h);
quit = 1;
break;
}
for(int tx_num = 1; tx_num <= num_txs; tx_num++)
{
try {
a >> tx;
}
catch (const std::exception& e)
{
LOG_PRINT_RED_L0("exception while de-archiving tx, height=" << h <<", tx_num=" << tx_num);
quit = 1;
break;
}
// if (tx_num == 1) {
// std::cout << "coinbase transaction" << ENDL;
// }
// crypto::hash hsh = null_hash;
// size_t blob_size = 0;
// NOTE: all tx hashes except for coinbase tx are available in the block data
// get_transaction_hash(tx, hsh, blob_size);
// LOG_PRINT_L0("tx " << tx_num << " " << hsh << " : " << ENDL);
// LOG_PRINT_L0(obj_to_json_str(tx) << ENDL);
// add blocks with verification.
// for Blockchain and blockchain_storage add_new_block().
if (opt_verify)
{
if (tx_num == 1) {
continue; // coinbase transaction. no need to insert to tx_pool.
}
// crypto::hash hsh = null_hash;
// size_t blob_size = 0;
// get_transaction_hash(tx, hsh, blob_size);
tx_verification_context tvc = AUTO_VAL_INIT(tvc);
bool r = true;
r = simple_core.m_pool.add_tx(tx, tvc, true);
if (!r)
{
LOG_PRINT_RED_L0("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num);
quit = 1;
break;
}
}
else
{
// for add_block() method, without (much) processing.
// don't add coinbase transaction to txs.
//
// because add_block() calls
// add_transaction(blk_hash, blk.miner_tx) first, and
// then a for loop for the transactions in txs.
if (tx_num > 1)
{
txs.push_back(tx);
}
}
}
if (opt_verify)
{
block_verification_context bvc = boost::value_initialized<block_verification_context>();
simple_core.m_storage.add_new_block(b, bvc);
if (bvc.m_verifivation_failed)
{
LOG_PRINT_L0("Failed to add block to blockchain, verification failed, height = " << h);
LOG_PRINT_L0("skipping rest of import file");
// ok to commit previously batched data because it failed only in
// verification of potential new block with nothing added to batch
// yet
quit = 1;
break;
}
if (! bvc.m_added_to_main_chain)
{
LOG_PRINT_L0("Failed to add block to blockchain, height = " << h);
LOG_PRINT_L0("skipping rest of import file");
// make sure we don't commit partial block data
quit = 2;
break;
}
}
else
{
size_t block_size;
difficulty_type cumulative_difficulty;
uint64_t coins_generated;
a >> block_size;
a >> cumulative_difficulty;
a >> coins_generated;
std::cout << "\r \r";
LOG_PRINT_L2("block_size: " << block_size);
LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty);
LOG_PRINT_L2("coins_generated: " << coins_generated);
try
{
simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs);
}
catch (const std::exception& e)
{
std::cout << "\r \r";
LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what());
quit = 2; // make sure we don't commit partial block data
break;
}
}
if (use_batch)
{
if (h % db_batch_size == 0)
{
std::cout << "\r \r";
std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL;
simple_core.batch_stop();
simple_core.batch_start();
std::cout << ENDL;
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
simple_core.m_storage.get_db()->show_stats();
#endif
}
}
}
}
catch (const std::exception& e)
{
std::cout << "\r \r";
LOG_PRINT_RED_L0("exception while reading from import file, height=" << h);
return 2;
}
} // while
import_file.close();
if (use_batch)
{
if (quit > 1)
{
// There was an error, so don't commit pending data.
// Destructor will abort write txn.
}
else
{
simple_core.batch_stop();
}
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
simple_core.m_storage.get_db()->show_stats();
#endif
if (h > 0)
LOG_PRINT_L0("Finished at height: " << h << " block: " << h-1);
}
std::cout << ENDL;
return 0;
}
int main(int argc, char* argv[])
{
std::string import_filename = BLOCKCHAIN_RAW;
#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
std::string default_db_engine = "memory";
#else
std::string default_db_engine = "lmdb";
#endif
uint32_t log_level = LOG_LEVEL_0;
std::string dirname;
std::string db_arg_str;
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level};
const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size};
const command_line::arg_descriptor<bool> arg_testnet_on = {
"testnet"
, "Run on testnet."
, false
};
const command_line::arg_descriptor<bool> arg_count_blocks = {
"count-blocks"
, "Count blocks in import file and exit"
, false
};
const command_line::arg_descriptor<std::string> arg_database = {
"database", "available: memory, lmdb"
, default_db_engine
};
const command_line::arg_descriptor<bool> arg_verify = {"verify",
"Verify blocks and transactions during import", true};
const command_line::arg_descriptor<bool> arg_batch = {"batch",
"Batch transactions for faster import", true};
const command_line::arg_descriptor<bool> arg_resume = {"resume",
"Resume from current height if output database already exists", true};
command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_batch_size);
command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_database);
command_line::add_arg(desc_cmd_only, arg_count_blocks);
command_line::add_arg(desc_cmd_only, command_line::arg_help);
// call add_options() directly for these arguments since
// command_line helpers support only boolean switch, not boolean argument
desc_cmd_sett.add_options()
(arg_verify.name, make_semantic(arg_verify), arg_verify.description)
(arg_batch.name, make_semantic(arg_batch), arg_batch.description)
(arg_resume.name, make_semantic(arg_resume), arg_resume.description)
;
po::options_description desc_options("Allowed options");
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
po::store(po::parse_command_line(argc, argv, desc_options), vm);
po::notify(vm);
return true;
});
if (! r)
return 1;
log_level = command_line::get_arg(vm, arg_log_level);
opt_verify = command_line::get_arg(vm, arg_verify);
opt_batch = command_line::get_arg(vm, arg_batch);
opt_resume = command_line::get_arg(vm, arg_resume);
db_batch_size = command_line::get_arg(vm, arg_batch_size);
if (command_line::get_arg(vm, command_line::arg_help))
{
std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL;
std::cout << desc_options << std::endl;
return 1;
}
if (! opt_batch && ! vm["batch-size"].defaulted())
{
std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL;
exit(1);
}
if (! db_batch_size)
{
std::cerr << "Error: batch-size must be > 0" << ENDL;
exit(1);
}
std::vector<std::string> db_engines {"memory", "lmdb"};
opt_testnet = command_line::get_arg(vm, arg_testnet_on);
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
dirname = command_line::get_arg(vm, data_dir_arg);
db_arg_str = command_line::get_arg(vm, arg_database);
log_space::get_set_log_detalisation_level(true, log_level);
log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
LOG_PRINT_L0("Starting...");
LOG_PRINT_L0("Setting log level = " << log_level);
boost::filesystem::path file_path {dirname};
std::string import_file_path;
import_file_path = (file_path / "export" / import_filename).string();
if (command_line::has_arg(vm, arg_count_blocks))
{
count_blocks(import_file_path);
exit(0);
}
std::string db_engine;
int mdb_flags = 0;
int res = 0;
res = parse_db_arguments(db_arg_str, db_engine, mdb_flags);
if (res)
{
std::cerr << "Error parsing database argument(s)" << ENDL;
exit(1);
}
if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end())
{
std::cerr << "Invalid database engine: " << db_engine << std::endl;
exit(1);
}
LOG_PRINT_L0("database: " << db_engine);
LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
if (opt_batch)
{
LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha
<< " batch size: " << db_batch_size);
}
else
{
LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha);
}
LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha);
LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
std::cout << "import file path: " << import_file_path << ENDL;
std::cout << "database path: " << file_path.string() << ENDL;
try
{
// fake_core needed for verification to work when enabled.
//
// NOTE: don't need fake_core method of doing things when we're going to call
// BlockchainDB add_block() directly and have available the 3 block
// properties to do so. Both ways work, but fake core isn't necessary in that
// circumstance.
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
if (db_engine == "lmdb")
{
fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags);
import_from_file(simple_core, import_file_path);
}
else if (db_engine == "memory")
{
fake_core_memory simple_core(dirname, opt_testnet);
import_from_file(simple_core, import_file_path);
}
else
{
std::cerr << "database engine unrecognized" << ENDL;
exit(1);
}
// for multi_db_compile:
#else
if (db_engine != default_db_engine)
{
std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl;
exit(1);
}
#if BLOCKCHAIN_DB == DB_LMDB
fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags);
#else
fake_core_memory simple_core(dirname, opt_testnet);
#endif
import_from_file(simple_core, import_file_path);
#endif
}
catch (const DB_ERROR& e)
{
std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL;
exit(1);
}
// destructors called at exit:
//
// ensure db closed
// - transactions properly checked and handled
// - disk sync if needed
//
// fake_core object's destructor is called when it goes out of scope. For an
// LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn
// calls delete on its BlockchainDB derived class' object, which closes its
// files.
}

View file

@ -0,0 +1,143 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <boost/filesystem.hpp>
#include "cryptonote_core/blockchain.h" // BlockchainDB and LMDB
#include "cryptonote_core/blockchain_storage.h" // in-memory DB
using namespace cryptonote;
#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB
struct fake_core_lmdb
{
Blockchain m_storage;
tx_memory_pool m_pool;
bool support_batch;
bool support_add_block;
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool)
// for multi_db_compile:
#else
fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool)
#endif
{
m_pool.init(path.string());
m_storage.init(path.string(), use_testnet, mdb_flags);
if (do_batch)
m_storage.get_db()->set_batch_transactions(do_batch);
support_batch = true;
support_add_block = true;
}
~fake_core_lmdb()
{
m_storage.deinit();
}
uint64_t add_block(const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<transaction>& txs
)
{
return m_storage.get_db()->add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
}
void batch_start()
{
m_storage.get_db()->batch_start();
}
void batch_stop()
{
m_storage.get_db()->batch_stop();
}
};
#endif
#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_MEMORY
struct fake_core_memory
{
blockchain_storage m_storage;
tx_memory_pool m_pool;
bool support_batch;
bool support_add_block;
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(&m_storage), m_storage(m_pool)
#else
// for multi_db_compile:
fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(m_storage), m_storage(&m_pool)
#endif
{
m_pool.init(path.string());
m_storage.init(path.string(), use_testnet);
support_batch = false;
support_add_block = false;
}
~fake_core_memory()
{
LOG_PRINT_L3("fake_core_memory() destructor called - want to see it ripple down");
m_storage.deinit();
}
uint64_t add_block(const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<transaction>& txs
)
{
// TODO:
// would need to refactor handle_block_to_main_chain() to have a direct add_block() method like Blockchain class
throw std::runtime_error("direct add_block() method not implemented for in-memory db");
return 2;
}
void batch_start()
{
LOG_PRINT_L0("WARNING: [batch_start] opt_batch set, but this database doesn't support/need transactions - ignoring");
}
void batch_stop()
{
LOG_PRINT_L0("WARNING: [batch_stop] opt_batch set, but this database doesn't support/need transactions - ignoring");
}
};
#endif

View file

@ -0,0 +1,37 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
// TODO: bounds checking is done before writing to buffer, but buffer size
// should be a sensible maximum
#define BUFFER_SIZE 1000000
#define NUM_BLOCKS_PER_CHUNK 1
#define STR_LENGTH_OF_INT 9
#define STR_FORMAT_OF_INT "%09d"
#define BLOCKCHAIN_RAW "blockchain.raw"

View file

@ -326,7 +326,7 @@ public:
void show_stats(); void show_stats();
// open the db at location <filename>, or create it if there isn't one. // open the db at location <filename>, or create it if there isn't one.
virtual void open(const std::string& filename) = 0; virtual void open(const std::string& filename, const int db_flags = 0) = 0;
// make sure implementation has a create function as well // make sure implementation has a create function as well
virtual void create(const std::string& filename) = 0; virtual void create(const std::string& filename) = 0;

View file

@ -149,6 +149,20 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
namespace cryptonote namespace cryptonote
{ {
// If m_batch_active is set, a batch transaction exists beyond this class, such
// as a batch import with verification enabled, or possibly (later) a batch
// network sync.
//
// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
// and get_tx(), when m_batch_active is set, the lookup uses the batch
// transaction. This isn't only because the transaction is available, but it's
// necessary so that lookups include the database updates only present in the
// current batch write.
//
// A regular network sync without batch writes is expected to open a new read
// transaction, as those lookups are part of the validation done prior to the
// write for block and tx data, so no write transaction is open at the time.
void BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::add_block( const block& blk
, const size_t& block_size , const size_t& block_size
, const difficulty_type& cumulative_difficulty , const difficulty_type& cumulative_difficulty
@ -621,7 +635,7 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
m_height = 0; m_height = 0;
} }
void BlockchainLMDB::open(const std::string& filename) void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -661,7 +675,7 @@ void BlockchainLMDB::open(const std::string& filename)
size_t mapsize = 1LL << 34; size_t mapsize = 1LL << 34;
if (auto result = mdb_env_set_mapsize(m_env, mapsize)) if (auto result = mdb_env_set_mapsize(m_env, mapsize))
throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str()));
if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0644)) if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str()));
// get a read/write MDB_txn // get a read/write MDB_txn
@ -1260,14 +1274,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open(); check_open();
// If m_batch_active is set, a batch transaction exists beyond this class,
// such as a batch import with verification enabled, or possibly (later) a
// batch network sync.
//
// A regular network sync without batching would be expected to open a new
// read transaction here, as validation is done prior to the write for block
// and tx data.
mdb_txn_safe txn; mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn; mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active) if (m_batch_active)

View file

@ -114,7 +114,7 @@ public:
BlockchainLMDB(bool batch_transactions=false); BlockchainLMDB(bool batch_transactions=false);
~BlockchainLMDB(); ~BlockchainLMDB();
virtual void open(const std::string& filename); virtual void open(const std::string& filename, const int mdb_flags=0);
virtual void create(const std::string& filename); virtual void create(const std::string& filename);

View file

@ -226,7 +226,7 @@ uint64_t Blockchain::get_current_blockchain_height() const
//------------------------------------------------------------------ //------------------------------------------------------------------
//FIXME: possibly move this into the constructor, to avoid accidentally //FIXME: possibly move this into the constructor, to avoid accidentally
// dereferencing a null BlockchainDB pointer // dereferencing a null BlockchainDB pointer
bool Blockchain::init(const std::string& config_folder, bool testnet) bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags)
{ {
LOG_PRINT_L3("Blockchain::" << __func__); LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock); CRITICAL_REGION_LOCAL(m_blockchain_lock);
@ -246,7 +246,7 @@ bool Blockchain::init(const std::string& config_folder, bool testnet)
const std::string filename = folder.string(); const std::string filename = folder.string();
try try
{ {
m_db->open(filename); m_db->open(filename, db_flags);
} }
catch (const DB_OPEN_FAILURE& e) catch (const DB_OPEN_FAILURE& e)
{ {

View file

@ -81,7 +81,7 @@ namespace cryptonote
Blockchain(tx_memory_pool& tx_pool); Blockchain(tx_memory_pool& tx_pool);
bool init(const std::string& config_folder, bool testnet = false); bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0);
bool deinit(); bool deinit();
void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; }
@ -145,6 +145,11 @@ namespace cryptonote
void set_enforce_dns_checkpoints(bool enforce); void set_enforce_dns_checkpoints(bool enforce);
bool update_checkpoints(const std::string& file_path, bool check_dns); bool update_checkpoints(const std::string& file_path, bool check_dns);
BlockchainDB* get_db()
{
return m_db;
}
private: private:
typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index; typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index;
typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container; typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container;