From b52abd1370cc21484d64f45504adbab47240debf Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 14 May 2017 14:06:55 +0100 Subject: [PATCH] Move txpool to the database Integration could go further (ie, return_tx_to_pool calls should not be needed anymore, possibly other things). poolstate.bin is now obsolete. --- src/blockchain_db/blockchain_db.h | 86 +++ src/blockchain_db/lmdb/db_lmdb.cpp | 216 +++++++ src/blockchain_db/lmdb/db_lmdb.h | 19 + .../blockchain_import.cpp | 2 +- src/blockchain_utilities/fake_core.h | 4 +- src/cryptonote_core/blockchain.cpp | 74 ++- src/cryptonote_core/blockchain.h | 13 +- src/cryptonote_core/cryptonote_core.cpp | 40 +- src/cryptonote_core/cryptonote_core.h | 15 +- src/cryptonote_core/tx_pool.cpp | 568 +++++++++++------- src/cryptonote_core/tx_pool.h | 52 +- .../cryptonote_protocol_handler.inl | 16 +- src/rpc/core_rpc_server.cpp | 6 +- tests/core_proxy/core_proxy.h | 3 +- tests/core_tests/chaingen.h | 10 + tests/unit_tests/ban.cpp | 3 +- tests/unit_tests/hardfork.cpp | 9 + 17 files changed, 847 insertions(+), 289 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index f5710550..9dd343f4 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -124,6 +124,27 @@ struct tx_data_t }; #pragma pack(pop) +/** + * @brief a struct containing txpool per transaction metadata + */ +struct txpool_tx_meta_t +{ + crypto::hash max_used_block_id; + crypto::hash last_failed_id; + uint64_t blob_size; + uint64_t fee; + uint64_t max_used_block_height; + uint64_t last_failed_height; + uint64_t receive_time; + uint64_t last_relayed_time; + // 112 bytes + uint8_t kept_by_block; + uint8_t relayed; + uint8_t do_not_relay; + + uint8_t padding[77]; // till 192 bytes +}; + /*********************************** * Exception Definitions ***********************************/ @@ -1251,6 +1272,71 @@ public: */ virtual bool has_key_image(const crypto::key_image& img) const = 0; + /** + * @brief add a txpool transaction + * + * @param details the details of the transaction to add + */ + virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0; + + /** + * @brief update a txpool transaction's metadata + * + * @param txid the txid of the transaction to update + * @param details the details of the transaction to update + */ + virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) = 0; + + /** + * @brief get the number of transactions in the txpool + */ + virtual uint64_t get_txpool_tx_count() const = 0; + + /** + * @brief check whether a txid is in the txpool + */ + virtual bool txpool_has_tx(const crypto::hash &txid) const = 0; + + /** + * @brief remove a txpool transaction + * + * @param txid the transaction id of the transation to remove + */ + virtual void remove_txpool_tx(const crypto::hash& txid) = 0; + + /** + * @brief get a txpool transaction's metadata + * + * @param txid the transaction id of the transation to lookup + * + * @return the metadata associated with that transaction + */ + virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const = 0; + + /** + * @brief get a txpool transaction's blob + * + * @param txid the transaction id of the transation to lookup + * + * @return the blob for that transaction + */ + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0; + + /** + * @brief runs a function over all txpool transactions + * + * The subclass should run the passed function for each txpool tx it has + * stored, passing the tx id and metadata as its parameters. + * + * If any call to the function returns false, the subclass should return + * false. Otherwise, the subclass returns true. + * + * @param std::function fn the function to run + * + * @return false if the function returns false for any transaction, otherwise true + */ + virtual bool for_all_txpool_txes(std::function, bool include_blob = false) const = 0; + /** * @brief runs a function over all key images stored * diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index dab09e8f..1f083187 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -169,6 +169,9 @@ int compare_string(const MDB_val *a, const MDB_val *b) * * spent_keys input hash - * + * txpool_meta txn hash txn metadata + * txpool_blob txn hash txn blob + * * Note: where the data items are of uniform size, DUPFIXED tables have * been used to save space. In most of these cases, a dummy "zerokval" * key is used when accessing the table; the Key listed above will be @@ -189,6 +192,9 @@ const char* const LMDB_OUTPUT_TXS = "output_txs"; const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; const char* const LMDB_SPENT_KEYS = "spent_keys"; +const char* const LMDB_TXPOOL_META = "txpool_meta"; +const char* const LMDB_TXPOOL_BLOB = "txpool_blob"; + const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; const char* const LMDB_HF_VERSIONS = "hf_versions"; @@ -1164,6 +1170,9 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); + lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta"); + lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob"); + // this subdb is dropped on sight, so it may not be present when we open the DB. // Since we use MDB_CREATE, we'll get an exception if we open read-only and it does not exist. // So we don't open for read-only, and also not drop below. It is not used elsewhere. @@ -1181,6 +1190,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); + mdb_set_compare(txn, m_txpool_meta, compare_hash32); + mdb_set_compare(txn, m_txpool_blob, compare_hash32); mdb_set_compare(txn, m_properties, compare_string); if (!(mdb_flags & MDB_RDONLY)) @@ -1429,6 +1440,211 @@ void BlockchainLMDB::unlock() auto_txn.commit(); \ } while(0) +void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(txpool_meta) + CURSOR(txpool_blob) + + const crypto::hash txid = get_transaction_hash(tx); + + MDB_val k = {sizeof(txid), (void *)&txid}; + MDB_val v = {sizeof(meta), (void *)&meta}; + if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) { + if (result == MDB_KEYEXIST) + throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db")); + else + throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str())); + } + MDB_val_copy blob_val(tx_to_blob(tx)); + if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) { + if (result == MDB_KEYEXIST) + throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db")); + else + throw1(DB_ERROR(lmdb_error("Error adding txpool tx blob to db transaction: ", result).c_str())); + } +} + +void BlockchainLMDB::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(txpool_meta) + CURSOR(txpool_blob) + + MDB_val k = {sizeof(txid), (void *)&txid}; + MDB_val v; + auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET); + if (result != 0) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to update: ", result).c_str())); + result = mdb_cursor_del(m_cur_txpool_meta, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str())); + v = MDB_val({sizeof(meta), (void *)&meta}); + if ((result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) != 0) { + if (result == MDB_KEYEXIST) + throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db")); + else + throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str())); + } +} + +uint64_t BlockchainLMDB::get_txpool_tx_count() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + int result; + + MDB_stat db_stats; + if ((result = mdb_stat(m_txn, m_txpool_meta, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txpool_meta: ", result).c_str())); + + TXN_POSTFIX_RDONLY(); + + return db_stats.ms_entries; +} + +bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(txpool_meta) + + MDB_val k = {sizeof(txid), (void *)&txid}; + auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str())); + TXN_POSTFIX_RDONLY(); + return result != MDB_NOTFOUND; +} + +void BlockchainLMDB::remove_txpool_tx(const crypto::hash& txid) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(txpool_meta) + CURSOR(txpool_blob) + + MDB_val k = {sizeof(txid), (void *)&txid}; + auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to remove: ", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_txpool_meta, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str())); + } + result = mdb_cursor_get(m_cur_txpool_blob, &k, NULL, MDB_SET); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx blob to remove: ", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_txpool_blob, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx blob to db transaction: ", result).c_str())); + } +} + +txpool_tx_meta_t BlockchainLMDB::get_txpool_tx_meta(const crypto::hash& txid) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(txpool_meta) + + MDB_val k = {sizeof(txid), (void *)&txid}; + MDB_val v; + auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET); + if (result != 0) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str())); + + const txpool_tx_meta_t meta = *(const txpool_tx_meta_t*)v.mv_data; + TXN_POSTFIX_RDONLY(); + return meta; +} + +cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(txpool_blob) + + MDB_val k = {sizeof(txid), (void *)&txid}; + MDB_val v; + auto result = mdb_cursor_get(m_cur_txpool_blob, &k, &v, MDB_SET); + if (result != 0) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str())); + + blobdata bd; + bd.assign(reinterpret_cast(v.mv_data), v.mv_size); + TXN_POSTFIX_RDONLY(); + return bd; +} + +bool BlockchainLMDB::for_all_txpool_txes(std::function f, bool include_blob) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(txpool_meta); + RCURSOR(txpool_blob); + + MDB_val k; + MDB_val v; + bool ret = true; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op); + op = MDB_NEXT; + if (result == MDB_NOTFOUND) + break; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str())); + const crypto::hash txid = *(const crypto::hash*)k.mv_data; + const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; + const cryptonote::blobdata *passed_bd = NULL; + cryptonote::blobdata bd; + if (include_blob) + { + MDB_val b; + result = mdb_cursor_get(m_cur_txpool_blob, &k, &b, MDB_SET); + if (result == MDB_NOTFOUND) + throw0(DB_ERROR("Failed to find txpool tx blob to match metadata")); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str())); + bd.assign(reinterpret_cast(b.mv_data), b.mv_size); + passed_bd = &bd; + } + + if (!f(txid, meta, passed_bd)) { + ret = false; + break; + } + } + + TXN_POSTFIX_RDONLY(); + + return ret; +} + bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8a567756..02f57ce1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -55,6 +55,9 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_spent_keys; + MDB_cursor *m_txc_txpool_meta; + MDB_cursor *m_txc_txpool_blob; + MDB_cursor *m_txc_hf_versions; } mdb_txn_cursors; @@ -67,6 +70,8 @@ typedef struct mdb_txn_cursors #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys +#define m_cur_txpool_meta m_cursors->m_txc_txpool_meta +#define m_cur_txpool_blob m_cursors->m_txc_txpool_blob #define m_cur_hf_versions m_cursors->m_txc_hf_versions typedef struct mdb_rflags @@ -81,6 +86,8 @@ typedef struct mdb_rflags bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; + bool m_rf_txpool_meta; + bool m_rf_txpool_blob; bool m_rf_hf_versions; } mdb_rflags; @@ -232,6 +239,15 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; + virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta); + virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta); + virtual uint64_t get_txpool_tx_count() const; + virtual bool txpool_has_tx(const crypto::hash &txid) const; + virtual void remove_txpool_tx(const crypto::hash& txid); + virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const; + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + virtual bool for_all_txpool_txes(std::function f, bool include_blob = false) const; + virtual bool for_all_key_images(std::function) const; virtual bool for_all_blocks(std::function) const; virtual bool for_all_transactions(std::function) const; @@ -364,6 +380,9 @@ private: MDB_dbi m_spent_keys; + MDB_dbi m_txpool_meta; + MDB_dbi m_txpool_blob; + MDB_dbi m_hf_starting_heights; MDB_dbi m_hf_versions; diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index b2c217ca..a43e2b4a 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -443,7 +443,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, // tx number 1: coinbase tx // tx number 2 onwards: archived_txs unsigned int tx_num = 1; - for (const transaction& tx : archived_txs) + for (transaction tx : archived_txs) { ++tx_num; // if tx is invalid diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h index 70144184..809c6fe4 100644 --- a/src/blockchain_utilities/fake_core.h +++ b/src/blockchain_utilities/fake_core.h @@ -59,8 +59,6 @@ struct fake_core_db // for multi_db_runtime: fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool) { - m_pool.init(path.string()); - BlockchainDB* db = nullptr; if (db_type == "lmdb") db = new BlockchainLMDB(); @@ -98,6 +96,8 @@ struct fake_core_db m_storage.init(db, m_hardfork, use_testnet); + m_pool.init(); + if (do_batch) m_storage.get_db().set_batch_transactions(do_batch); support_batch = true; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 0d5a8d46..c85c5308 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3047,7 +3047,7 @@ bool Blockchain::check_block_timestamp(const block& b) const return check_block_timestamp(timestamps, b); } //------------------------------------------------------------------ -void Blockchain::return_tx_to_pool(const std::vector &txs) +void Blockchain::return_tx_to_pool(std::vector &txs) { uint8_t version = get_current_hard_fork_version(); for (auto& tx : txs) @@ -3565,7 +3565,7 @@ void Blockchain::block_longhash_worker(const uint64_t height, const std::vector< bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) { MTRACE("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); + CRITICAL_REGION_BEGIN(m_blockchain_lock); TIME_MEASURE_START(t1); m_db->batch_stop(); @@ -3601,6 +3601,9 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) m_blocks_txs_check.clear(); m_check_txin_table.clear(); + CRITICAL_REGION_END(); + m_tx_pool.unlock(); + return true; } @@ -3634,14 +3637,32 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::listbatch_start(blocks_entry.size()))) { m_blockchain_lock.unlock(); + m_tx_pool.unlock(); epee::misc_utils::sleep_no_w(1000); + m_tx_pool.lock(); m_blockchain_lock.lock(); } @@ -3957,6 +3978,41 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::listadd_txpool_tx(tx, meta); +} + +void Blockchain::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta) +{ + m_db->update_txpool_tx(txid, meta); +} + +void Blockchain::remove_txpool_tx(const crypto::hash &txid) +{ + m_db->remove_txpool_tx(txid); +} + +uint64_t Blockchain::get_txpool_tx_count() const +{ + return m_db->get_txpool_tx_count(); +} + +txpool_tx_meta_t Blockchain::get_txpool_tx_meta(const crypto::hash& txid) const +{ + return m_db->get_txpool_tx_meta(txid); +} + +cryptonote::blobdata Blockchain::get_txpool_tx_blob(const crypto::hash& txid) const +{ + return m_db->get_txpool_tx_blob(txid); +} + +bool Blockchain::for_all_txpool_txes(std::function f, bool include_blob) const +{ + return m_db->for_all_txpool_txes(f, include_blob); +} + void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) { m_db_sync_mode = sync_mode; @@ -4071,6 +4127,8 @@ void Blockchain::load_compiled_in_block_hashes() // The core will not call check_tx_inputs(..) for these // transactions in this case. Consequently, the sanity check // for tx hashes will fail in handle_block_to_main_chain(..) + CRITICAL_REGION_LOCAL(m_tx_pool); + std::list txs; m_tx_pool.get_transactions(txs); @@ -4089,6 +4147,16 @@ void Blockchain::load_compiled_in_block_hashes() } #endif +void Blockchain::lock() +{ + m_blockchain_lock.lock(); +} + +void Blockchain::unlock() +{ + m_blockchain_lock.unlock(); +} + bool Blockchain::for_all_key_images(std::function f) const { return m_db->for_all_key_images(f); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 9dacba36..100430c3 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -854,6 +854,17 @@ namespace cryptonote */ std::list> get_alternative_chains() const; + void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta); + void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); + void remove_txpool_tx(const crypto::hash &txid); + uint64_t get_txpool_tx_count() const; + txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const; + cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + bool for_all_txpool_txes(std::function, bool include_blob = false) const; + + void lock(); + void unlock(); + void cancel(); private: @@ -1238,7 +1249,7 @@ namespace cryptonote * @return true */ bool update_next_cumulative_size_limit(); - void return_tx_to_pool(const std::vector &txs); + void return_tx_to_pool(std::vector &txs); /** * @brief make sure a transaction isn't attempting a double-spend diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 4d4b9d4d..5a3e4d1f 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -272,9 +272,6 @@ namespace cryptonote m_config_folder_mempool = m_config_folder_mempool + "/" + m_port; } - r = m_mempool.init(m_fakechain ? std::string() : m_config_folder_mempool); - CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); - std::string db_type = command_line::get_arg(vm, command_line::arg_db_type); std::string db_sync_mode = command_line::get_arg(vm, command_line::arg_db_sync_mode); bool fast_sync = command_line::get_arg(vm, command_line::arg_fast_block_sync) != 0; @@ -401,6 +398,9 @@ namespace cryptonote r = m_blockchain_storage.init(db, m_testnet, test_options); + r = m_mempool.init(); + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); + // now that we have a valid m_blockchain_storage, we can clean out any // transactions in the pool that do not conform to the current fork m_mempool.validate(m_blockchain_storage.get_current_hard_fork_version()); @@ -760,7 +760,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { crypto::hash tx_hash = get_transaction_hash(tx); crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); @@ -774,7 +774,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if(m_mempool.have_tx(tx_hash)) { @@ -795,17 +795,15 @@ namespace cryptonote bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::list> txs; + std::list> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); NOTIFY_NEW_TRANSACTIONS::request r; - blobdata bl; for (auto it = txs.begin(); it != txs.end(); ++it) { - t_serializable_object_to_blob(it->second, bl); - r.txs.push_back(bl); + r.txs.push_back(it->second); } get_protocol()->relay_transactions(r, fake_context); m_mempool.set_relayed(txs); @@ -815,7 +813,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) { - std::list> txs; + std::list> txs; cryptonote::transaction tx; crypto::hash tx_hash, tx_prefix_hash; if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) @@ -823,7 +821,7 @@ namespace cryptonote LOG_ERROR("Failed to parse relayed transaction"); return; } - txs.push_back(std::make_pair(tx_hash, std::move(tx))); + txs.push_back(std::make_pair(tx_hash, std::move(tx_blob))); m_mempool.set_relayed(txs); } //----------------------------------------------------------------------------------------------- @@ -893,9 +891,9 @@ namespace cryptonote bce.block = cryptonote::block_to_blob(b); for (const auto &tx_hash: b.tx_hashes) { - cryptonote::transaction tx; - CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, tx), "Transaction not found in pool"); - bce.txs.push_back(cryptonote::tx_to_blob(tx)); + cryptonote::blobdata txblob; + CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, txblob), "Transaction not found in pool"); + bce.txs.push_back(txblob); } return bce; } @@ -955,6 +953,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::prepare_handle_incoming_blocks(const std::list &blocks) { + m_incoming_tx_lock.lock(); m_blockchain_storage.prepare_handle_incoming_blocks(blocks); return true; } @@ -962,7 +961,11 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::cleanup_handle_incoming_blocks(bool force_sync) { - m_blockchain_storage.cleanup_handle_incoming_blocks(force_sync); + try { + m_blockchain_storage.cleanup_handle_incoming_blocks(force_sync); + } + catch (...) {} + m_incoming_tx_lock.unlock(); return true; } @@ -1043,11 +1046,16 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transaction(const crypto::hash &id, transaction& tx) const + bool core::get_pool_transaction(const crypto::hash &id, cryptonote::blobdata& tx) const { return m_mempool.get_transaction(id, tx); } //----------------------------------------------------------------------------------------------- + bool core::pool_has_tx(const crypto::hash &id) const + { + return m_mempool.have_tx(id); + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions_and_spent_keys_info(std::vector& tx_infos, std::vector& key_image_infos) const { return m_mempool.get_transactions_and_spent_keys_info(tx_infos, key_image_infos); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 24b0f061..286b7228 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -389,6 +389,13 @@ namespace cryptonote */ void set_enforce_dns_checkpoints(bool enforce_dns); + /** + * @copydoc tx_memory_pool::have_tx + * + * @note see tx_memory_pool::have_tx + */ + bool pool_has_tx(const crypto::hash &txid) const; + /** * @copydoc tx_memory_pool::get_transactions * @@ -408,7 +415,7 @@ namespace cryptonote * * @note see tx_memory_pool::get_transaction */ - bool get_pool_transaction(const crypto::hash& id, transaction& tx) const; + bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx) const; /** * @copydoc tx_memory_pool::get_pool_transactions_and_spent_keys_info @@ -658,7 +665,7 @@ namespace cryptonote private: /** - * @copydoc add_new_tx(const transaction&, tx_verification_context&, bool) + * @copydoc add_new_tx(transaction&, tx_verification_context&, bool) * * @param tx_hash the transaction's hash * @param tx_prefix_hash the transaction prefix' hash @@ -667,7 +674,7 @@ namespace cryptonote * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -684,7 +691,7 @@ namespace cryptonote * is already in a block on the Blockchain, or is successfully added * to the transaction pool */ - bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @copydoc Blockchain::add_new_block diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 37b6fb4b..e9c5c820 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -38,6 +38,7 @@ #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_config.h" #include "blockchain.h" +#include "blockchain_db/blockchain_db.h" #include "common/boost_serialization_helper.h" #include "common/int-util.h" #include "misc_language.h" @@ -64,7 +65,7 @@ namespace cryptonote float const ACCEPT_THRESHOLD = 1.0f; // a kind of increasing backoff within min/max bounds - time_t get_relay_delay(time_t now, time_t received) + uint64_t get_relay_delay(time_t now, time_t received) { time_t d = (now - received + MIN_RELAY_TIME) / MIN_RELAY_TIME * MIN_RELAY_TIME; if (d > MAX_RELAY_TIME) @@ -81,6 +82,21 @@ namespace cryptonote { return get_min_block_size(version) * 125 / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } + + // This class is meant to create a batch when none currently exists. + // If a batch exists, it can't be from another thread, since we can + // only be called with the txpool lock taken, and it is held during + // the whole prepare/handle/cleanup incoming block sequence. + class LockedTXN { + public: + LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false) { + m_batch = m_blockchain.get_db().batch_start(); + } + ~LockedTXN() { if (m_batch) { m_blockchain.get_db().batch_stop(); } } + private: + Blockchain &m_blockchain; + bool m_batch; + }; } //--------------------------------------------------------------------------------- //--------------------------------------------------------------------------------- @@ -89,7 +105,7 @@ namespace cryptonote } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(const transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { PERF_TIMER(add_tx); if (tx.version == 0) @@ -187,29 +203,38 @@ namespace cryptonote crypto::hash max_used_block_id = null_hash; uint64_t max_used_block_height = 0; - tx_details txd; - txd.tx = tx; - bool ch_inp_res = m_blockchain.check_tx_inputs(txd.tx, max_used_block_height, max_used_block_id, tvc, kept_by_block); - CRITICAL_REGION_LOCAL(m_transactions_lock); + cryptonote::txpool_tx_meta_t meta; + bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block); if(!ch_inp_res) { // if the transaction was valid before (kept_by_block), then it // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - auto txd_p = m_transactions.insert(transactions_container::value_type(id, txd)); - CHECK_AND_ASSERT_MES(txd_p.second, false, "transaction already exists at inserting in memory pool"); - txd_p.first->second.blob_size = blob_size; - txd_p.first->second.fee = fee; - txd_p.first->second.max_used_block_id = null_hash; - txd_p.first->second.max_used_block_height = 0; - txd_p.first->second.last_failed_height = 0; - txd_p.first->second.last_failed_id = null_hash; - txd_p.first->second.kept_by_block = kept_by_block; - txd_p.first->second.receive_time = receive_time; - txd_p.first->second.last_relayed_time = time(NULL); - txd_p.first->second.relayed = relayed; - txd_p.first->second.do_not_relay = do_not_relay; + meta.blob_size = blob_size; + meta.fee = fee; + meta.max_used_block_id = null_hash; + meta.max_used_block_height = 0; + meta.last_failed_height = 0; + meta.last_failed_id = null_hash; + meta.kept_by_block = kept_by_block; + meta.receive_time = receive_time; + meta.last_relayed_time = time(NULL); + meta.relayed = relayed; + meta.do_not_relay = do_not_relay; + try + { + LockedTXN lock(m_blockchain); + m_blockchain.add_txpool_tx(tx, meta); + if (!insert_key_images(tx, kept_by_block)) + return false; + m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)blob_size, receive_time), id); + } + catch (const std::exception &e) + { + MERROR("transaction already exists at inserting in memory pool: " << e.what()); + return false; + } tvc.m_verifivation_impossible = true; tvc.m_added_to_pool = true; }else @@ -221,30 +246,60 @@ namespace cryptonote }else { //update transactions container - auto txd_p = m_transactions.insert(transactions_container::value_type(id, txd)); - CHECK_AND_ASSERT_MES(txd_p.second, false, "internal error: transaction already exists at inserting in memorypool"); - txd_p.first->second.blob_size = blob_size; - txd_p.first->second.kept_by_block = kept_by_block; - txd_p.first->second.fee = fee; - txd_p.first->second.max_used_block_id = max_used_block_id; - txd_p.first->second.max_used_block_height = max_used_block_height; - txd_p.first->second.last_failed_height = 0; - txd_p.first->second.last_failed_id = null_hash; - txd_p.first->second.receive_time = receive_time; - txd_p.first->second.last_relayed_time = time(NULL); - txd_p.first->second.relayed = relayed; - txd_p.first->second.do_not_relay = do_not_relay; + meta.blob_size = blob_size; + meta.kept_by_block = kept_by_block; + meta.fee = fee; + meta.max_used_block_id = max_used_block_id; + meta.max_used_block_height = max_used_block_height; + meta.last_failed_height = 0; + meta.last_failed_id = null_hash; + meta.receive_time = receive_time; + meta.last_relayed_time = time(NULL); + meta.relayed = relayed; + meta.do_not_relay = do_not_relay; + + try + { + LockedTXN lock(m_blockchain); + m_blockchain.remove_txpool_tx(get_transaction_hash(tx)); + m_blockchain.add_txpool_tx(tx, meta); + if (!insert_key_images(tx, kept_by_block)) + return false; + m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)blob_size, receive_time), id); + } + catch (const std::exception &e) + { + MERROR("internal error: transaction already exists at inserting in memorypool: " << e.what()); + return false; + } tvc.m_added_to_pool = true; - if(txd_p.first->second.fee > 0 && !do_not_relay) + if(meta.fee > 0 && !do_not_relay) tvc.m_should_be_relayed = true; } // assume failure during verification steps until success is certain tvc.m_verifivation_failed = true; + tvc.m_verifivation_failed = false; + + MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); + return true; + } + //--------------------------------------------------------------------------------- + bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay, uint8_t version) + { + crypto::hash h = null_hash; + size_t blob_size = 0; + get_transaction_hash(tx, h, blob_size); + return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + } + //--------------------------------------------------------------------------------- + bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block) + { for(const auto& in: tx.vin) { + const crypto::hash id = get_transaction_hash(tx); CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false); std::unordered_set& kei_image_set = m_spent_key_images[txin.k_image]; CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block @@ -253,29 +308,16 @@ namespace cryptonote auto ins_res = kei_image_set.insert(id); CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set"); } - - tvc.m_verifivation_failed = false; - - m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)blob_size, receive_time), id); - - MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay, uint8_t version) - { - crypto::hash h = null_hash; - size_t blob_size = 0; - get_transaction_hash(tx, h, blob_size); - return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); - } - //--------------------------------------------------------------------------------- //FIXME: Can return early before removal of all of the key images. // At the least, need to make sure that a false return here // is treated properly. Should probably not return early, however. bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx) { CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); // ND: Speedup // 1. Move transaction hash calcuation outside of loop. ._. crypto::hash actual_hash = get_transaction_hash(tx); @@ -306,22 +348,37 @@ namespace cryptonote bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay) { CRITICAL_REGION_LOCAL(m_transactions_lock); - auto it = m_transactions.find(id); - if(it == m_transactions.end()) - return false; + CRITICAL_REGION_LOCAL1(m_blockchain); auto sorted_it = find_tx_in_sorted_container(id); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) return false; - tx = it->second.tx; - blob_size = it->second.blob_size; - fee = it->second.fee; - relayed = it->second.relayed; - do_not_relay = it->second.do_not_relay; - remove_transaction_keyimages(it->second.tx); - m_transactions.erase(it); + try + { + LockedTXN lock(m_blockchain); + txpool_tx_meta_t meta = m_blockchain.get_txpool_tx_meta(id); + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id); + if (!parse_and_validate_tx_from_blob(txblob, tx)) + { + MERROR("Failed to parse tx from txpool"); + return false; + } + blob_size = meta.blob_size; + fee = meta.fee; + relayed = meta.relayed; + do_not_relay = meta.do_not_relay; + + // remove first, in case this throws, so key images aren't removed + m_blockchain.remove_txpool_tx(id); + remove_transaction_keyimages(tx); + } + catch (const std::exception &e) + { + MERROR("Failed to remove tx from txpool: " << e.what()); + return false; + } + m_txs_by_fee_and_receive_time.erase(sorted_it); return true; } @@ -344,68 +401,113 @@ namespace cryptonote bool tx_memory_pool::remove_stuck_transactions() { CRITICAL_REGION_LOCAL(m_transactions_lock); - for(auto it = m_transactions.begin(); it!= m_transactions.end();) - { - uint64_t tx_age = time(nullptr) - it->second.receive_time; + CRITICAL_REGION_LOCAL1(m_blockchain); + std::unordered_set remove; + m_blockchain.for_all_txpool_txes([this, &remove](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + uint64_t tx_age = time(nullptr) - meta.receive_time; - if((tx_age > CRYPTONOTE_MEMPOOL_TX_LIVETIME && !it->second.kept_by_block) || - (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && it->second.kept_by_block) ) + if((tx_age > CRYPTONOTE_MEMPOOL_TX_LIVETIME && !meta.kept_by_block) || + (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) ) { - LOG_PRINT_L1("Tx " << it->first << " removed from tx pool due to outdated, age: " << tx_age ); - remove_transaction_keyimages(it->second.tx); - auto sorted_it = find_tx_in_sorted_container(it->first); + LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age ); + auto sorted_it = find_tx_in_sorted_container(txid); if (sorted_it == m_txs_by_fee_and_receive_time.end()) { - LOG_PRINT_L1("Removing tx " << it->first << " from tx pool, but it was not found in the sorted txs container!"); + LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); } else { m_txs_by_fee_and_receive_time.erase(sorted_it); } - m_timed_out_transactions.insert(it->first); - auto pit = it++; - m_transactions.erase(pit); - }else - ++it; + m_timed_out_transactions.insert(txid); + remove.insert(txid); + } + return true; + }); + + if (!remove.empty()) + { + LockedTXN lock(m_blockchain); + for (const crypto::hash &txid: remove) + { + try + { + cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid); + cryptonote::transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + { + MERROR("Failed to parse tx from txpool"); + // continue + } + else + { + // remove first, so we only remove key images if the tx removal succeeds + m_blockchain.remove_txpool_tx(txid); + remove_transaction_keyimages(tx); + } + } + catch (const std::exception &e) + { + MWARNING("Failed to remove stuck transaction: " << txid); + // ignore error + } + } } return true; } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::list> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::list> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - const time_t now = time(NULL); - for(auto it = m_transactions.begin(); it!= m_transactions.end();) - { + CRITICAL_REGION_LOCAL1(m_blockchain); + const uint64_t now = time(NULL); + m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed - if(it->second.fee > 0 && !it->second.do_not_relay && now - it->second.last_relayed_time > get_relay_delay(now, it->second.receive_time)) + if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) { // if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem // mentioned by smooth where nodes would flush txes at slightly different times, causing // flushed txes to be re-added when received from a node which was just about to flush it - time_t max_age = it->second.kept_by_block ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME; - if (now - it->second.receive_time <= max_age / 2) + uint64_t max_age = meta.kept_by_block ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME; + if (now - meta.receive_time <= max_age / 2) { - txs.push_back(std::make_pair(it->first, it->second.tx)); + try + { + cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid); + txs.push_back(std::make_pair(txid, bd)); + } + catch (const std::exception &e) + { + MERROR("Failed to get transaction blob from db"); + // ignore error + } } } - ++it; - } + return true; + }); return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::list> &txs) + void tx_memory_pool::set_relayed(const std::list> &txs) { CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); const time_t now = time(NULL); + LockedTXN lock(m_blockchain); for (auto it = txs.begin(); it != txs.end(); ++it) { - auto i = m_transactions.find(it->first); - if (i != m_transactions.end()) + try { - i->second.relayed = true; - i->second.last_relayed_time = now; + txpool_tx_meta_t meta = m_blockchain.get_txpool_tx_meta(it->first); + meta.relayed = true; + meta.last_relayed_time = now; + m_blockchain.update_txpool_tx(it->first, meta); + } + catch (const std::exception &e) + { + MERROR("Failed to update txpool transaction metadata: " << e.what()); + // continue } } } @@ -413,46 +515,67 @@ namespace cryptonote size_t tx_memory_pool::get_transactions_count() const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_transactions.size(); + CRITICAL_REGION_LOCAL1(m_blockchain); + return m_blockchain.get_txpool_tx_count(); } //--------------------------------------------------------------------------------- void tx_memory_pool::get_transactions(std::list& txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - for(const auto& tx_vt: m_transactions) - txs.push_back(tx_vt.second.tx); + CRITICAL_REGION_LOCAL1(m_blockchain); + m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + transaction tx; + if (!parse_and_validate_tx_from_blob(*bd, tx)) + { + MERROR("Failed to parse tx from txpool"); + // continue + return true; + } + txs.push_back(tx); + return true; + }, true); } //------------------------------------------------------------------ void tx_memory_pool::get_transaction_hashes(std::vector& txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - for(const auto& tx_vt: m_transactions) - txs.push_back(get_transaction_hash(tx_vt.second.tx)); + CRITICAL_REGION_LOCAL1(m_blockchain); + m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + txs.push_back(txid); + return true; + }); } //------------------------------------------------------------------ //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector& tx_infos, std::vector& key_image_infos) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - for (const auto& tx_vt : m_transactions) - { + CRITICAL_REGION_LOCAL1(m_blockchain); + m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; - const tx_details& txd = tx_vt.second; - txi.id_hash = epee::string_tools::pod_to_hex(tx_vt.first); - txi.tx_json = obj_to_json_str(*const_cast(&txd.tx)); - txi.blob_size = txd.blob_size; - txi.fee = txd.fee; - txi.kept_by_block = txd.kept_by_block; - txi.max_used_block_height = txd.max_used_block_height; - txi.max_used_block_id_hash = epee::string_tools::pod_to_hex(txd.max_used_block_id); - txi.last_failed_height = txd.last_failed_height; - txi.last_failed_id_hash = epee::string_tools::pod_to_hex(txd.last_failed_id); - txi.receive_time = txd.receive_time; - txi.relayed = txd.relayed; - txi.last_relayed_time = txd.last_relayed_time; - txi.do_not_relay = txd.do_not_relay; + txi.id_hash = epee::string_tools::pod_to_hex(txid); + transaction tx; + if (!parse_and_validate_tx_from_blob(*bd, tx)) + { + MERROR("Failed to parse tx from txpool"); + // continue + return true; + } + txi.tx_json = obj_to_json_str(tx); + txi.blob_size = meta.blob_size; + txi.fee = meta.fee; + txi.kept_by_block = meta.kept_by_block; + txi.max_used_block_height = meta.max_used_block_height; + txi.max_used_block_id_hash = epee::string_tools::pod_to_hex(meta.max_used_block_id); + txi.last_failed_height = meta.last_failed_height; + txi.last_failed_id_hash = epee::string_tools::pod_to_hex(meta.last_failed_id); + txi.receive_time = meta.receive_time; + txi.relayed = meta.relayed; + txi.last_relayed_time = meta.last_relayed_time; + txi.do_not_relay = meta.do_not_relay; tx_infos.push_back(txi); - } + return true; + }, true); for (const key_images_container::value_type& kee : m_spent_key_images) { const crypto::key_image& k_image = kee.first; @@ -468,14 +591,19 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_transaction(const crypto::hash& id, transaction& tx) const + bool tx_memory_pool::get_transaction(const crypto::hash& id, cryptonote::blobdata& txblob) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - auto it = m_transactions.find(id); - if(it == m_transactions.end()) + CRITICAL_REGION_LOCAL1(m_blockchain); + try + { + txblob = m_blockchain.get_txpool_tx_blob(id); + return true; + } + catch (const std::exception &e) + { return false; - tx = it->second.tx; - return true; + } } //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) @@ -491,14 +619,14 @@ namespace cryptonote bool tx_memory_pool::have_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - if(m_transactions.count(id)) - return true; - return false; + CRITICAL_REGION_LOCAL1(m_blockchain); + return m_blockchain.get_db().txpool_has_tx(id); } //--------------------------------------------------------------------------------- bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx) const { CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); for(const auto& in: tx.vin) { CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail @@ -524,7 +652,7 @@ namespace cryptonote m_transactions_lock.unlock(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(tx_details& txd) const + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const { //not the best implementation at this time, sorry :( //check is ring_signature already checked ? @@ -535,7 +663,7 @@ namespace cryptonote return false;//we already sure that this tx is broken for this height tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -552,7 +680,7 @@ namespace cryptonote return false; //check ring signature again, it is possible (with very small chance) that this transaction become again valid tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -561,7 +689,7 @@ namespace cryptonote } } //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure - if(m_blockchain.have_tx_keyimges_as_spent(txd.tx)) + if(m_blockchain.have_tx_keyimges_as_spent(tx)) return false; //transaction is ok. @@ -594,20 +722,27 @@ namespace cryptonote { std::stringstream ss; CRITICAL_REGION_LOCAL(m_transactions_lock); - for (const transactions_container::value_type& txe : m_transactions) { - const tx_details& txd = txe.second; - ss << "id: " << txe.first << std::endl; + CRITICAL_REGION_LOCAL1(m_blockchain); + m_blockchain.for_all_txpool_txes([&ss, short_format](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *txblob) { + ss << "id: " << txid << std::endl; if (!short_format) { - ss << obj_to_json_str(*const_cast(&txd.tx)) << std::endl; + cryptonote::transaction tx; + if (!parse_and_validate_tx_from_blob(*txblob, tx)) + { + MERROR("Failed to parse tx from txpool"); + return true; // continue + } + ss << obj_to_json_str(tx) << std::endl; } - ss << "blob_size: " << txd.blob_size << std::endl - << "fee: " << print_money(txd.fee) << std::endl - << "kept_by_block: " << (txd.kept_by_block ? 'T' : 'F') << std::endl - << "max_used_block_height: " << txd.max_used_block_height << std::endl - << "max_used_block_id: " << txd.max_used_block_id << std::endl - << "last_failed_height: " << txd.last_failed_height << std::endl - << "last_failed_id: " << txd.last_failed_id << std::endl; - } + ss << "blob_size: " << meta.blob_size << std::endl + << "fee: " << print_money(meta.fee) << std::endl + << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl + << "max_used_block_height: " << meta.max_used_block_height << std::endl + << "max_used_block_id: " << meta.max_used_block_id << std::endl + << "last_failed_height: " << meta.last_failed_height << std::endl + << "last_failed_id: " << meta.last_failed_id << std::endl; + return true; + }, !short_format); return ss.str(); } @@ -620,6 +755,7 @@ namespace cryptonote // with it. CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); uint64_t best_coinbase = 0, coinbase = 0; total_size = 0; @@ -638,11 +774,11 @@ namespace cryptonote auto sorted_it = m_txs_by_fee_and_receive_time.begin(); while (sorted_it != m_txs_by_fee_and_receive_time.end()) { - auto tx_it = m_transactions.find(sorted_it->second); - LOG_PRINT_L2("Considering " << tx_it->first << ", size " << tx_it->second.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); + txpool_tx_meta_t meta = m_blockchain.get_txpool_tx_meta(sorted_it->second); + LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); // Can not exceed maximum block size - if (max_total_size < total_size + tx_it->second.blob_size) + if (max_total_size < total_size + meta.blob_size) { LOG_PRINT_L2(" would exceed maximum block size"); sorted_it++; @@ -655,13 +791,13 @@ namespace cryptonote // If we're getting lower coinbase tx, // stop including more tx uint64_t block_reward; - if(!get_block_reward(median_size, total_size + tx_it->second.blob_size, already_generated_coins, block_reward, version)) + if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version)) { LOG_PRINT_L2(" would exceed maximum block size"); sorted_it++; continue; } - coinbase = block_reward + fee + tx_it->second.fee; + coinbase = block_reward + fee + meta.fee; if (coinbase < template_accept_threshold(best_coinbase)) { LOG_PRINT_L2(" would decrease coinbase to " << print_money(coinbase)); @@ -680,27 +816,36 @@ namespace cryptonote } } + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second); + cryptonote::transaction tx; + if (!parse_and_validate_tx_from_blob(txblob, tx)) + { + MERROR("Failed to parse tx from txpool"); + sorted_it++; + continue; + } + // Skip transactions that are not ready to be // included into the blockchain or that are // missing key images - if (!is_transaction_ready_to_go(tx_it->second)) + if (!is_transaction_ready_to_go(meta, tx)) { LOG_PRINT_L2(" not ready to go"); sorted_it++; continue; } - if (have_key_images(k_images, tx_it->second.tx)) + if (have_key_images(k_images, tx)) { LOG_PRINT_L2(" key images already seen"); sorted_it++; continue; } - bl.tx_hashes.push_back(tx_it->first); - total_size += tx_it->second.blob_size; - fee += tx_it->second.fee; + bl.tx_hashes.push_back(sorted_it->second); + total_size += meta.blob_size; + fee += meta.fee; best_coinbase = coinbase; - append_key_images(k_images, tx_it->second.tx); + append_key_images(k_images, tx); sorted_it++; LOG_PRINT_L2(" added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase)); } @@ -714,103 +859,88 @@ namespace cryptonote size_t tx_memory_pool::validate(uint8_t version) { CRITICAL_REGION_LOCAL(m_transactions_lock); - size_t n_removed = 0; + CRITICAL_REGION_LOCAL1(m_blockchain); size_t tx_size_limit = get_transaction_size_limit(version); - for (auto it = m_transactions.begin(); it != m_transactions.end(); ) { - bool remove = false; - const crypto::hash &txid = get_transaction_hash(it->second.tx); - if (it->second.blob_size >= tx_size_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << it->second.blob_size << " bytes), removing it from pool"); - remove = true; + std::unordered_set remove; + + m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + if (meta.blob_size >= tx_size_limit) { + LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool"); + remove.insert(txid); } else if (m_blockchain.have_tx(txid)) { LOG_PRINT_L1("Transaction " << txid << " is in the blockchain, removing it from pool"); - remove = true; + remove.insert(txid); } - if (remove) { - remove_transaction_keyimages(it->second.tx); - auto sorted_it = find_tx_in_sorted_container(txid); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) + return true; + }); + + size_t n_removed = 0; + if (!remove.empty()) + { + LockedTXN lock(m_blockchain); + for (const crypto::hash &txid: remove) + { + try { - LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); + cryptonote::transaction tx; + if (!parse_and_validate_tx_from_blob(txblob, tx)) + { + MERROR("Failed to parse tx from txpool"); + continue; + } + // remove tx from db first + m_blockchain.remove_txpool_tx(txid); + remove_transaction_keyimages(tx); + auto sorted_it = find_tx_in_sorted_container(txid); + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + ++n_removed; } - else + catch (const std::exception &e) { - m_txs_by_fee_and_receive_time.erase(sorted_it); + MERROR("Failed to remove invalid tx from pool"); + // continue } - auto pit = it++; - m_transactions.erase(pit); - ++n_removed; - continue; } - it++; } return n_removed; } //--------------------------------------------------------------------------------- - //TODO: investigate whether only ever returning true is correct - bool tx_memory_pool::init(const std::string& config_folder) + bool tx_memory_pool::init() { CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); - m_config_folder = config_folder; - if (m_config_folder.empty()) + m_txs_by_fee_and_receive_time.clear(); + m_spent_key_images.clear(); + return m_blockchain.for_all_txpool_txes([this](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd) { + cryptonote::transaction tx; + if (!parse_and_validate_tx_from_blob(*bd, tx)) + { + MERROR("Failed to parse tx from txpool"); + return false; + } + if (!insert_key_images(tx, meta.kept_by_block)) + { + MFATAL("Failed to insert key images from txpool tx"); + return false; + } + m_txs_by_fee_and_receive_time.emplace(std::pair(meta.fee / (double)meta.blob_size, meta.receive_time), txid); return true; - - std::string state_file_path = config_folder + "/" + CRYPTONOTE_POOLDATA_FILENAME; - boost::system::error_code ec; - if(!boost::filesystem::exists(state_file_path, ec)) - return true; - bool res = tools::unserialize_obj_from_file(*this, state_file_path); - if(!res) - { - LOG_ERROR("Failed to load memory pool from file " << state_file_path); - - m_transactions.clear(); - m_txs_by_fee_and_receive_time.clear(); - m_spent_key_images.clear(); - } - - // no need to store queue of sorted transactions, as it's easy to generate. - for (const auto& tx : m_transactions) - { - m_txs_by_fee_and_receive_time.emplace(std::pair(tx.second.fee / (double)tx.second.blob_size, tx.second.receive_time), tx.first); - } - - // Ignore deserialization error - return true; + }, true); } //--------------------------------------------------------------------------------- - //TODO: investigate whether only ever returning true is correct bool tx_memory_pool::deinit() { - LOG_PRINT_L1("Received signal to deactivate memory pool store"); - - if (m_config_folder.empty()) - { - LOG_PRINT_L1("Memory pool store already empty"); - return true; - } - - if (!tools::create_directories_if_necessary(m_config_folder)) - { - LOG_ERROR("Failed to create memory pool data directory: " << m_config_folder); - return false; - } - - std::string state_file_path = m_config_folder + "/" + CRYPTONOTE_POOLDATA_FILENAME; - bool res = tools::serialize_obj_to_file(*this, state_file_path); - if(!res) - { - LOG_ERROR("Failed to serialize memory pool to file " << state_file_path); - return false; - } - else - { - LOG_PRINT_L1("Memory pool store deactivated successfully"); - return true; - } - + return true; } } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index d19f83b2..89526d82 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -43,6 +43,7 @@ #include "math_helper.h" #include "cryptonote_basic/cryptonote_basic_impl.h" #include "cryptonote_basic/verification_context.h" +#include "blockchain_db/blockchain_db.h" #include "crypto/hash.h" #include "rpc/core_rpc_server_commands_defs.h" @@ -100,12 +101,12 @@ namespace cryptonote /** - * @copydoc add_tx(const transaction&, tx_verification_context&, bool, bool, uint8_t) + * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash * @param blob_size the transaction's size */ - bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -124,7 +125,7 @@ namespace cryptonote * * @return true if the transaction passes validations, otherwise false */ - bool add_tx(const transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief takes a transaction with the given hash from the pool @@ -199,7 +200,7 @@ namespace cryptonote * * @return true */ - bool init(const std::string& config_folder); + bool init(); /** * @brief attempts to save the transaction pool state to disk @@ -256,11 +257,11 @@ namespace cryptonote * @brief get a specific transaction from the pool * * @param h the hash of the transaction to get - * @param tx return-by-reference the transaction requested + * @param tx return-by-reference the transaction blob requested * * @return true if the transaction is found, otherwise false */ - bool get_transaction(const crypto::hash& h, transaction& tx) const; + bool get_transaction(const crypto::hash& h, cryptonote::blobdata& txblob) const; /** * @brief get a list of all relayable transactions and their hashes @@ -274,14 +275,14 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::list>& txs) const; + bool get_relayable_transactions(std::list>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * * @param txs the list of transactions (and their hashes) */ - void set_relayed(const std::list>& txs); + void set_relayed(const std::list>& txs); /** * @brief get the total number of transactions in the pool @@ -316,28 +317,6 @@ namespace cryptonote #define CURRENT_MEMPOOL_ARCHIVE_VER 11 #define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 12 - /** - * @brief serialize the transaction pool to/from disk - * - * If the archive version passed is older than the version compiled - * in, this function does nothing, as it cannot deserialize after a - * format change. - * - * @tparam archive_t the archive class - * @param a the archive to serialize to/from - * @param version the archive version - */ - template - void serialize(archive_t & a, const unsigned int version) - { - if(version < CURRENT_MEMPOOL_ARCHIVE_VER ) - return; - CRITICAL_REGION_LOCAL(m_transactions_lock); - a & m_transactions; - a & m_spent_key_images; - a & m_timed_out_transactions; - } - /** * @brief information about a single transaction */ @@ -378,6 +357,13 @@ namespace cryptonote private: + /** + * @brief insert key images into m_spent_key_images + * + * @return true on success, false on error + */ + bool insert_key_images(const transaction &tx, bool kept_by_block); + /** * @brief remove old transactions from the pool * @@ -452,10 +438,7 @@ namespace cryptonote * * @return true if the transaction is good to go, otherwise false */ - bool is_transaction_ready_to_go(tx_details& txd) const; - - //! map transactions (and related info) by their hashes - typedef std::unordered_map transactions_container; + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const; //TODO: confirm the below comments and investigate whether or not this // is the desired behavior @@ -472,7 +455,6 @@ namespace cryptonote public: #endif mutable epee::critical_section m_transactions_lock; //!< lock for the pool - transactions_container m_transactions; //!< container for transactions in the pool #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) private: #endif diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 79578a34..8b41ce51 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -479,7 +479,7 @@ namespace cryptonote // we might already have the tx that the peer // sent in our pool, so don't verify again.. - if(!m_core.get_pool_transaction(tx_hash, tx)) + if(!m_core.pool_has_tx(tx_hash)) { MDEBUG("Incoming tx " << tx_hash << " not in pool, adding"); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -536,9 +536,10 @@ namespace cryptonote size_t tx_idx = 0; for(auto& tx_hash: new_block.tx_hashes) { - if(m_core.get_pool_transaction(tx_hash, tx)) + cryptonote::blobdata txblob; + if(m_core.get_pool_transaction(tx_hash, txblob)) { - have_tx.push_back(tx_to_blob(tx)); + have_tx.push_back(txblob); } else { @@ -731,6 +732,15 @@ namespace cryptonote if(context.m_state != cryptonote_connection_context::state_normal) return 1; + // while syncing, core will lock for a long time, so we ignore + // those txes as they aren't really needed anyway, and avoid a + // long block before replying + if(!is_synchronized()) + { + LOG_DEBUG_CC(context, "Received new tx while syncing, ignored"); + return 1; + } + for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 7cce0bf0..89093ded 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1608,13 +1608,13 @@ namespace cryptonote } crypto::hash txid = *reinterpret_cast(txid_data.data()); - cryptonote::transaction tx; - bool r = m_core.get_pool_transaction(txid, tx); + cryptonote::blobdata txblob; + bool r = m_core.get_pool_transaction(txid, txblob); if (r) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); NOTIFY_NEW_TRANSACTIONS::request r; - r.txs.push_back(cryptonote::tx_to_blob(tx)); + r.txs.push_back(txblob); m_core.get_protocol()->relay_transactions(r, fake_context); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes } diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 106c62f2..58645edb 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -90,7 +90,8 @@ namespace tests size_t get_block_sync_size() const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {} bool get_testnet() const { return false; } - bool get_pool_transaction(const crypto::hash& id, cryptonote::transaction& tx) const { return false; } + bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } + bool pool_has_tx(const crypto::hash &txid) const { return false; } bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const { return false; } bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const { return false; } bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; } diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index eef10312..add48eeb 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -472,6 +472,16 @@ inline bool do_replay_events(std::vector& events) MERROR("Failed to init core"); return false; } + + // start with a clean pool + std::vector pool_txs; + if (!c.get_pool_transaction_hashes(pool_txs)) + { + MERROR("Failed to flush txpool"); + return false; + } + c.get_blockchain_storage().flush_txes_from_pool(std::list(pool_txs.begin(), pool_txs.end())); + t_test_class validator; bool ret = replay_events_through_core(c, events, validator); c.deinit(); diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 595085d7..9816adb9 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -65,7 +65,8 @@ public: size_t get_block_sync_size() const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {} bool get_testnet() const { return false; } - bool get_pool_transaction(const crypto::hash& id, cryptonote::transaction& tx) const { return false; } + bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } + bool pool_has_tx(const crypto::hash &txid) const { return false; } bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const { return false; } bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const { return false; } bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; } diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 08c4276c..e29919e8 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -112,6 +112,15 @@ public: virtual bool is_read_only() const { return false; } virtual std::map> get_output_histogram(const std::vector &amounts, bool unlocked, uint64_t recent_cutoff) const { return std::map>(); } + virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) {} + virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) {} + virtual uint64_t get_txpool_tx_count() const { return 0; } + virtual bool txpool_has_tx(const crypto::hash &txid) const { return false; } + virtual void remove_txpool_tx(const crypto::hash& txid) {} + virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const { return txpool_tx_meta_t(); } + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const { return ""; } + virtual bool for_all_txpool_txes(std::function, bool include_blob = false) const { return false; } + virtual void add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty