Merge branch 'performance' into master

This commit is contained in:
Howard Chu 2016-04-05 21:13:16 +01:00
commit d7ea7d9a23
7 changed files with 658 additions and 775 deletions

View file

@ -6771,8 +6771,8 @@ set1:
if (op == MDB_GET_BOTH || rc > 0) if (op == MDB_GET_BOTH || rc > 0)
return MDB_NOTFOUND; return MDB_NOTFOUND;
rc = 0; rc = 0;
*data = olddata;
} }
*data = olddata;
} else { } else {
if (mc->mc_xcursor) if (mc->mc_xcursor)

View file

@ -82,14 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
} }
} }
add_transaction_data(blk_hash, tx, tx_hash); uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash);
std::vector<uint64_t> amount_output_indices;
// iterate tx.vout using indices instead of C++11 foreach syntax because // iterate tx.vout using indices instead of C++11 foreach syntax because
// we need the index // we need the index
for (uint64_t i = 0; i < tx.vout.size(); ++i) for (uint64_t i = 0; i < tx.vout.size(); ++i)
{ {
add_output(tx_hash, tx.vout[i], i, tx.unlock_time); amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time));
} }
add_tx_amount_output_indices(tx_id, amount_output_indices);
} }
uint64_t BlockchainDB::add_block( const block& blk uint64_t BlockchainDB::add_block( const block& blk

View file

@ -59,6 +59,39 @@
* Unspent transaction outputs are duplicated to quickly gather random * Unspent transaction outputs are duplicated to quickly gather random
* outputs to use for mixins * outputs to use for mixins
* *
* Indices and Identifiers:
* The word "index" is used ambiguously throughout this code. It is
* particularly confusing when talking about the output or transaction
* tables since their indexing can refer to themselves or each other.
* I have attempted to clarify these usages here:
*
* Blocks, transactions, and outputs are all identified by a hash.
* For storage efficiency, a 64-bit integer ID is used instead of the hash
* inside the DB. Tables exist to map between hash and ID. A block ID is
* also referred to as its "height". Transactions and outputs generally are
* not referred to by ID outside of this module, but the tx ID is returned
* by tx_exists() and used by get_tx_amount_output_indices(). Like their
* corresponding hashes, IDs are globally unique.
*
* The remaining uses of the word "index" refer to local offsets, and are
* not globally unique. An "amount output index" N refers to the Nth output
* of a specific amount. An "output local index" N refers to the Nth output
* of a specific tx.
*
* Exceptions:
* DB_ERROR -- generic
* DB_OPEN_FAILURE
* DB_CREATE_FAILURE
* DB_SYNC_FAILURE
* BLOCK_DNE
* BLOCK_PARENT_DNE
* BLOCK_EXISTS
* BLOCK_INVALID -- considering making this multiple errors
* TX_DNE
* TX_EXISTS
* OUTPUT_DNE
* OUTPUT_EXISTS
* KEY_IMAGE_EXISTS
*/ */
namespace cryptonote namespace cryptonote
@ -80,6 +113,15 @@ struct output_data_t
}; };
#pragma pack(pop) #pragma pack(pop)
#pragma pack(push, 1)
struct tx_data_t
{
uint64_t tx_id;
uint64_t unlock_time;
uint64_t block_id;
};
#pragma pack(pop)
/*********************************** /***********************************
* Exception Definitions * Exception Definitions
***********************************/ ***********************************/
@ -311,14 +353,18 @@ private:
* and the other data passed here, not the separate outputs of the * and the other data passed here, not the separate outputs of the
* transaction. * transaction.
* *
* It returns a tx ID, which is a mapping from the tx_hash. The tx ID
* is used in #add_tx_amount_output_indices().
*
* If any of this cannot be done, the subclass should throw the corresponding * If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION * subclass of DB_EXCEPTION
* *
* @param blk_hash the hash of the block containing the transaction * @param blk_hash the hash of the block containing the transaction
* @param tx the transaction to be added * @param tx the transaction to be added
* @param tx_hash the hash of the transaction * @param tx_hash the hash of the transaction
* @return the transaction ID
*/ */
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0;
/** /**
* @brief remove data about a transaction * @brief remove data about a transaction
@ -348,6 +394,9 @@ private:
* future, this tracking (of the number, at least) should be moved to * future, this tracking (of the number, at least) should be moved to
* this class, as it is necessary and the same among all BlockchainDB. * this class, as it is necessary and the same among all BlockchainDB.
* *
* It returns an amount output index, which is the index of the output
* for its specified amount.
*
* This data should be stored in such a manner that the only thing needed to * This data should be stored in such a manner that the only thing needed to
* reverse the process is the tx_out. * reverse the process is the tx_out.
* *
@ -358,25 +407,24 @@ private:
* @param tx_output the output * @param tx_output the output
* @param local_index index of the output in its transaction * @param local_index index of the output in its transaction
* @param unlock_time unlock time/height of the output * @param unlock_time unlock time/height of the output
* @return amount output index
*/ */
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
/** /**
* @brief remove an output * @brief store amount output indices for a tx's outputs
* *
* The subclass implementing this will remove all output data it stored * The subclass implementing this will add the amount output indices to its
* in add_output(). * backing store in a suitable manner. The tx_id will be the same one that
* * was returned from #add_output().
* In addition, the subclass is responsible for correctly decrementing
* its global output counter (this may be automatic for some, such as using
* a database backend "count" feature).
* *
* If any of this cannot be done, the subclass should throw the corresponding * If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION * subclass of DB_EXCEPTION
* *
* @param tx_output the output to be removed * @param tx_id ID of the transaction containing these outputs
* @param amount_output_indices the amount output indices of the transaction
*/ */
virtual void remove_output(const tx_out& tx_output) = 0; virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0;
/** /**
* @brief store a spent key * @brief store a spent key
@ -930,10 +978,12 @@ public:
* given hash and return true if so, false otherwise. * given hash and return true if so, false otherwise.
* *
* @param h the hash to check against * @param h the hash to check against
* @param tx_id (optional) returns the tx_id for the tx hash
* *
* @return true if the transaction exists, otherwise false * @return true if the transaction exists, otherwise false
*/ */
virtual bool tx_exists(const crypto::hash& h) const = 0; virtual bool tx_exists(const crypto::hash& h) const = 0;
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0;
// return unlock time of tx with hash <h> // return unlock time of tx with hash <h>
/** /**
@ -1124,37 +1174,21 @@ public:
*/ */
virtual bool can_thread_bulk_indices() const = 0; virtual bool can_thread_bulk_indices() const = 0;
/**
* @brief gets output indices (global) for a transaction's outputs
*
* The subclass should fetch the global output indices for each output
* in the transaction with the given hash.
*
* If the transaction does not exist, the subclass should throw TX_DNE.
*
* If an output cannot be found, the subclass should throw OUTPUT_DNE.
*
* @param h a transaction hash
*
* @return a list of global output indices
*/
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0;
/** /**
* @brief gets output indices (amount-specific) for a transaction's outputs * @brief gets output indices (amount-specific) for a transaction's outputs
* *
* The subclass should fetch the amount-specific output indices for each * The subclass should fetch the amount-specific output indices for each
* output in the transaction with the given hash. * output in the transaction with the given ID.
* *
* If the transaction does not exist, the subclass should throw TX_DNE. * If the transaction does not exist, the subclass should throw TX_DNE.
* *
* If an output cannot be found, the subclass should throw OUTPUT_DNE. * If an output cannot be found, the subclass should throw OUTPUT_DNE.
* *
* @param h a transaction hash * @param tx_id a transaction ID
* *
* @return a list of amount-specific output indices * @return a list of amount-specific output indices
*/ */
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0; virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0;
/** /**
* @brief check if a key image is stored as spent * @brief check if a key image is stored as spent

File diff suppressed because it is too large Load diff

View file

@ -43,20 +43,13 @@ typedef struct mdb_txn_cursors
{ {
MDB_cursor *m_txc_blocks; MDB_cursor *m_txc_blocks;
MDB_cursor *m_txc_block_heights; MDB_cursor *m_txc_block_heights;
MDB_cursor *m_txc_block_hashes; MDB_cursor *m_txc_block_info;
MDB_cursor *m_txc_block_timestamps;
MDB_cursor *m_txc_block_sizes;
MDB_cursor *m_txc_block_diffs;
MDB_cursor *m_txc_block_coins;
MDB_cursor *m_txc_output_txs; MDB_cursor *m_txc_output_txs;
MDB_cursor *m_txc_output_indices;
MDB_cursor *m_txc_output_amounts; MDB_cursor *m_txc_output_amounts;
MDB_cursor *m_txc_output_keys;
MDB_cursor *m_txc_txs; MDB_cursor *m_txc_txs;
MDB_cursor *m_txc_tx_heights; MDB_cursor *m_txc_tx_indices;
MDB_cursor *m_txc_tx_unlocks;
MDB_cursor *m_txc_tx_outputs; MDB_cursor *m_txc_tx_outputs;
MDB_cursor *m_txc_spent_keys; MDB_cursor *m_txc_spent_keys;
@ -66,18 +59,11 @@ typedef struct mdb_txn_cursors
#define m_cur_blocks m_cursors->m_txc_blocks #define m_cur_blocks m_cursors->m_txc_blocks
#define m_cur_block_heights m_cursors->m_txc_block_heights #define m_cur_block_heights m_cursors->m_txc_block_heights
#define m_cur_block_hashes m_cursors->m_txc_block_hashes #define m_cur_block_info m_cursors->m_txc_block_info
#define m_cur_block_timestamps m_cursors->m_txc_block_timestamps
#define m_cur_block_sizes m_cursors->m_txc_block_sizes
#define m_cur_block_diffs m_cursors->m_txc_block_diffs
#define m_cur_block_coins m_cursors->m_txc_block_coins
#define m_cur_output_txs m_cursors->m_txc_output_txs #define m_cur_output_txs m_cursors->m_txc_output_txs
#define m_cur_output_indices m_cursors->m_txc_output_indices
#define m_cur_output_amounts m_cursors->m_txc_output_amounts #define m_cur_output_amounts m_cursors->m_txc_output_amounts
#define m_cur_output_keys m_cursors->m_txc_output_keys
#define m_cur_txs m_cursors->m_txc_txs #define m_cur_txs m_cursors->m_txc_txs
#define m_cur_tx_heights m_cursors->m_txc_tx_heights #define m_cur_tx_indices m_cursors->m_txc_tx_indices
#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks
#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs
#define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_spent_keys m_cursors->m_txc_spent_keys
#define m_cur_hf_versions m_cursors->m_txc_hf_versions #define m_cur_hf_versions m_cursors->m_txc_hf_versions
@ -87,18 +73,11 @@ typedef struct mdb_rflags
bool m_rf_txn; bool m_rf_txn;
bool m_rf_blocks; bool m_rf_blocks;
bool m_rf_block_heights; bool m_rf_block_heights;
bool m_rf_block_hashes; bool m_rf_block_info;
bool m_rf_block_timestamps;
bool m_rf_block_sizes;
bool m_rf_block_diffs;
bool m_rf_block_coins;
bool m_rf_output_txs; bool m_rf_output_txs;
bool m_rf_output_indices;
bool m_rf_output_amounts; bool m_rf_output_amounts;
bool m_rf_output_keys;
bool m_rf_txs; bool m_rf_txs;
bool m_rf_tx_heights; bool m_rf_tx_indices;
bool m_rf_tx_unlocks;
bool m_rf_tx_outputs; bool m_rf_tx_outputs;
bool m_rf_spent_keys; bool m_rf_spent_keys;
bool m_rf_hf_versions; bool m_rf_hf_versions;
@ -223,6 +202,7 @@ public:
virtual uint64_t height() const; virtual uint64_t height() const;
virtual bool tx_exists(const crypto::hash& h) const; virtual bool tx_exists(const crypto::hash& h) const;
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const;
virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
@ -246,10 +226,8 @@ public:
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices); virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
virtual bool has_key_image(const crypto::key_image& img) const; virtual bool has_key_image(const crypto::key_image& img) const;
@ -306,18 +284,23 @@ private:
virtual void remove_block(); virtual void remove_block();
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time); virtual uint64_t add_output(const crypto::hash& tx_hash,
const tx_out& tx_output,
const uint64_t& local_index,
const uint64_t unlock_time
);
virtual void remove_output(const tx_out& tx_output); virtual void add_tx_amount_output_indices(const uint64_t tx_id,
const std::vector<uint64_t>& amount_output_indices
);
void remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx); void remove_tx_outputs(const uint64_t tx_id, const transaction& tx);
void remove_output(const MDB_val *out_index, const uint64_t amount); void remove_output(const uint64_t amount, const uint64_t& out_index);
void remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index);
virtual void add_spent_key(const crypto::key_image& k_image); virtual void add_spent_key(const crypto::key_image& k_image);
@ -349,16 +332,6 @@ private:
*/ */
tx_out output_from_blob(const blobdata& blob) const; tx_out output_from_blob(const blobdata& blob) const;
/**
* @brief get the global index of the index-th output of the given amount
*
* @param amount the output amount
* @param index the index into the set of outputs of that amount
*
* @return the global index of the desired output
*/
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
void check_open() const; void check_open() const;
virtual bool is_read_only() const; virtual bool is_read_only() const;
@ -370,21 +343,14 @@ private:
MDB_dbi m_blocks; MDB_dbi m_blocks;
MDB_dbi m_block_heights; MDB_dbi m_block_heights;
MDB_dbi m_block_hashes; MDB_dbi m_block_info;
MDB_dbi m_block_timestamps;
MDB_dbi m_block_sizes;
MDB_dbi m_block_diffs;
MDB_dbi m_block_coins;
MDB_dbi m_txs; MDB_dbi m_txs;
MDB_dbi m_tx_unlocks; MDB_dbi m_tx_indices;
MDB_dbi m_tx_heights;
MDB_dbi m_tx_outputs; MDB_dbi m_tx_outputs;
MDB_dbi m_output_txs; MDB_dbi m_output_txs;
MDB_dbi m_output_indices;
MDB_dbi m_output_amounts; MDB_dbi m_output_amounts;
MDB_dbi m_output_keys;
MDB_dbi m_spent_keys; MDB_dbi m_spent_keys;
@ -394,6 +360,7 @@ private:
MDB_dbi m_properties; MDB_dbi m_properties;
uint64_t m_height; uint64_t m_height;
uint64_t m_num_txs;
uint64_t m_num_outputs; uint64_t m_num_outputs;
mutable uint64_t m_cum_size; // used in batch size estimation mutable uint64_t m_cum_size; // used in batch size estimation
mutable int m_cum_count; mutable int m_cum_count;

View file

@ -1970,14 +1970,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
{ {
LOG_PRINT_L3("Blockchain::" << __func__); LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock); CRITICAL_REGION_LOCAL(m_blockchain_lock);
if (!m_db->tx_exists(tx_id)) uint64_t tx_index;
if (!m_db->tx_exists(tx_id, tx_index))
{ {
LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id);
return false; return false;
} }
// get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts
indexs = m_db->get_tx_amount_output_indices(tx_id); indexs = m_db->get_tx_amount_output_indices(tx_index);
CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty");
return true; return true;

View file

@ -78,6 +78,7 @@ public:
virtual block get_top_block() const { return block(); } virtual block get_top_block() const { return block(); }
virtual uint64_t height() const { return blocks.size(); } virtual uint64_t height() const { return blocks.size(); }
virtual bool tx_exists(const crypto::hash& h) const { return false; } virtual bool tx_exists(const crypto::hash& h) const { return false; }
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; }
virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; } virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; }
virtual transaction get_tx(const crypto::hash& h) const { return transaction(); } virtual transaction get_tx(const crypto::hash& h) const { return transaction(); }
virtual uint64_t get_tx_count() const { return 0; } virtual uint64_t get_tx_count() const { return 0; }
@ -93,13 +94,13 @@ public:
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) {} virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) {}
virtual bool can_thread_bulk_indices() const { return false; } virtual bool can_thread_bulk_indices() const { return false; }
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); } virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); }
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); } virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); }
virtual bool has_key_image(const crypto::key_image& img) const { return false; } virtual bool has_key_image(const crypto::key_image& img) const { return false; }
virtual void remove_block() { blocks.pop_back(); } virtual void remove_block() { blocks.pop_back(); }
virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {} virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;}
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {} virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {}
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {} virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) {return 0;}
virtual void remove_output(const tx_out& tx_output) {} virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) {}
virtual void add_spent_key(const crypto::key_image& k_image) {} virtual void add_spent_key(const crypto::key_image& k_image) {}
virtual void remove_spent_key(const crypto::key_image& k_image) {} virtual void remove_spent_key(const crypto::key_image& k_image) {}