kumi/danicoin
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

core, wallet: faster tx pool scanning

Browse source 
Includes a new RPC to get tx pool hashes fast.
This commit is contained in:
moneromooo-monero 2017-03-22 18:03:23 +00:00
parent f065234b71
commit 558cfc31ca
No known key found for this signature in database
GPG key ID: 686F07454D6CEFC3
8 changed files with 166 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/cryptonote_core/cryptonote_core.cpp
View file

@ -1036,7 +1036,13 @@ namespace cryptonote
m_mempool.get_transactions(txs);
return true;
}
//-----------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------
bool core::get_pool_transaction_hashes(std::vector<crypto::hash>& txs) const
{
m_mempool.get_transaction_hashes(txs);
return true;
}
//-----------------------------------------------------------------------------------------------
bool core::get_pool_transaction(const crypto::hash &id, transaction& tx) const
{
return m_mempool.get_transaction(id, tx);

7
src/cryptonote_core/cryptonote_core.h
View file

@ -396,6 +396,13 @@ namespace cryptonote
*/
bool get_pool_transactions(std::list<transaction>& txs) const;
/**
* @copydoc tx_memory_pool::get_transactions
*
* @note see tx_memory_pool::get_transactions
*/
bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs) const;
/**
* @copydoc tx_memory_pool::get_transaction
*

7
src/cryptonote_core/tx_pool.cpp
View file

@ -423,6 +423,13 @@ namespace cryptonote
txs.push_back(tx_vt.second.tx);
}
//------------------------------------------------------------------
void tx_memory_pool::get_transaction_hashes(std::vector<crypto::hash>& txs) const
{
CRITICAL_REGION_LOCAL(m_transactions_lock);
for(const auto& tx_vt: m_transactions)
txs.push_back(get_transaction_hash(tx_vt.second.tx));
}
//------------------------------------------------------------------
//TODO: investigate whether boolean return is appropriate
bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
{

7
src/cryptonote_core/tx_pool.h
View file

@ -233,6 +233,13 @@ namespace cryptonote
*/
void get_transactions(std::list<transaction>& txs) const;
/**
* @brief get a list of all transaction hashes in the pool
*
* @param txs return-by-reference the list of transactions
*/
void get_transaction_hashes(std::vector<crypto::hash>& txs) const;
/**
* @brief get information about all transactions and key images in the pool
*

8
src/rpc/core_rpc_server.cpp
View file

@ -789,6 +789,14 @@ namespace cryptonote
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res)
{
CHECK_CORE_BUSY();
m_core.get_pool_transaction_hashes(res.tx_hashes);
res.status = CORE_RPC_STATUS_OK;
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res)
{
// FIXME: replace back to original m_p2p.send_stop_signal() after

2
src/rpc/core_rpc_server.h
View file

@ -92,6 +92,7 @@ namespace cryptonote
MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted)
MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted)
MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL)
MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES)
MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted)
MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO)
MAP_URI_AUTO_JON2_IF("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS, !m_restricted)
@ -145,6 +146,7 @@ namespace cryptonote
bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res);
bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res);
bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res);
bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res);
bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res);
bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res);
bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res);

22
src/rpc/core_rpc_server_commands_defs.h
View file

@ -49,7 +49,7 @@ namespace cryptonote
// advance which version they will stop working with
// Don't go over 32767 for any of these
#define CORE_RPC_VERSION_MAJOR 1
#define CORE_RPC_VERSION_MINOR 7
#define CORE_RPC_VERSION_MINOR 8
#define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
#define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
@ -1025,6 +1025,26 @@ namespace cryptonote
};
};
struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES
{
struct request
{
BEGIN_KV_SERIALIZE_MAP()
END_KV_SERIALIZE_MAP()
};
struct response
{
std::string status;
std::vector<crypto::hash> tx_hashes;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(status)
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(tx_hashes)
END_KV_SERIALIZE_MAP()
};
};
struct COMMAND_RPC_GET_CONNECTIONS
{
struct request

209
src/wallet/wallet2.cpp
View file

@ -1384,13 +1384,13 @@ void wallet2::update_pool_state()
MDEBUG("update_pool_state start");
// get the pool state
cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req;
cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res;
cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request req;
cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response res;
m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_json("/get_transaction_pool", req, res, m_http_client, rpc_timeout);
bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout);
m_daemon_rpc_mutex.unlock();
THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool");
THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool");
THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool_hashes.bin");
THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool_hashes.bin");
THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error);
MDEBUG("update_pool_state got pool");
@ -1398,11 +1398,11 @@ void wallet2::update_pool_state()
std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin();
while (it != m_unconfirmed_txs.end())
{
const std::string txid = epee::string_tools::pod_to_hex(it->first);
const crypto::hash &txid = it->first;
bool found = false;
for (auto it2: res.transactions)
for (const auto &it2: res.tx_hashes)
{
if (it2.id_hash == txid)
if (it2 == txid)
{
found = true;
break;
@ -1453,11 +1453,11 @@ void wallet2::update_pool_state()
std::unordered_map<crypto::hash, wallet2::payment_details>::iterator uit = m_unconfirmed_payments.begin();
while (uit != m_unconfirmed_payments.end())
{
const std::string txid = string_tools::pod_to_hex(uit->first);
const crypto::hash &txid = uit->first;
bool found = false;
for (auto it2: res.transactions)
for (const auto &it2: res.tx_hashes)
{
if (it2.id_hash == txid)
if (it2 == txid)
{
found = true;
break;
@ -1466,114 +1466,119 @@ void wallet2::update_pool_state()
auto pit = uit++;
if (!found)
{
MDEBUG("Removing " << txid << " from unconfirmed payments, not found in pool");
m_unconfirmed_payments.erase(pit);
}
}
MDEBUG("update_pool_state done second loop");
// add new pool txes to us
for (auto it: res.transactions)
// gather txids of new pool txes to us
std::vector<crypto::hash> txids;
for (const auto &txid: res.tx_hashes)
{
cryptonote::blobdata txid_data;
if(epee::string_tools::parse_hexstr_to_binbuff(it.id_hash, txid_data) && txid_data.size() == sizeof(crypto::hash))
if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end())
{
const crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end())
LOG_PRINT_L2("Already seen " << txid << ", skipped");
continue;
}
if (m_unconfirmed_payments.find(txid) == m_unconfirmed_payments.end())
{
LOG_PRINT_L1("Found new pool tx: " << txid);
bool found = false;
for (const auto &i: m_unconfirmed_txs)
{
LOG_PRINT_L2("Already seen " << txid << ", skipped");
continue;
if (i.first == txid)
{
found = true;
break;
}
}
if (m_unconfirmed_payments.find(txid) == m_unconfirmed_payments.end())
if (!found)
{
LOG_PRINT_L1("Found new pool tx: " << txid);
bool found = false;
for (const auto &i: m_unconfirmed_txs)
{
if (i.first == txid)
{
found = true;
break;
}
}
if (!found)
{
// not one of those we sent ourselves
cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req;
cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res;
req.txs_hashes.push_back(it.id_hash);
MDEBUG("asking for " << it.id_hash);
req.decode_as_json = false;
m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
MDEBUG("asked for " << it.id_hash << ", got " << r << " and " << res.status);
m_daemon_rpc_mutex.unlock();
if (r && res.status == CORE_RPC_STATUS_OK)
{
if (res.txs.size() == 1)
{
// might have just been put in a block
if (res.txs[0].in_pool)
{
cryptonote::transaction tx;
cryptonote::blobdata bd;
crypto::hash tx_hash, tx_prefix_hash;
if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd))
{
if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash))
{
if (tx_hash == txid)
{
process_new_transaction(txid, tx, std::vector<uint64_t>(), 0, time(NULL), false, true);
m_scanned_pool_txs[0].insert(txid);
if (m_scanned_pool_txs[0].size() > 5000)
{
std::swap(m_scanned_pool_txs[0], m_scanned_pool_txs[1]);
m_scanned_pool_txs[0].clear();
}
}
else
{
LOG_PRINT_L0("Mismatched txids when processing unconfimed txes from pool");
}
}
else
{
LOG_PRINT_L0("failed to validate transaction from daemon");
}
}
else
{
LOG_PRINT_L0("Failed to parse tx " << txid);
}
}
else
{
LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more");
}
}
else
{
LOG_PRINT_L0("Expected 1 tx, got " << res.txs.size());
}
}
else
{
LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status);
}
}
else
{
LOG_PRINT_L1("We sent that one");
}
// not one of those we sent ourselves
txids.push_back(txid);
}
else
{
LOG_PRINT_L1("Already saw that one, it's for us");
LOG_PRINT_L1("We sent that one");
}
}
else
{
LOG_PRINT_L0("Failed to parse txid");
LOG_PRINT_L1("Already saw that one, it's for us");
}
}
// get those txes
if (!txids.empty())
{
cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req;
cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res;
for (const auto &txid: txids)
req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
MDEBUG("asking for " << txids.size() << " transactions");
req.decode_as_json = false;
m_daemon_rpc_mutex.lock();
bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
m_daemon_rpc_mutex.unlock();
MDEBUG("Got " << r << " and " << res.status);
if (r && res.status == CORE_RPC_STATUS_OK)
{
if (res.txs.size() == txids.size())
{
size_t n = 0;
for (const auto &txid: txids)
{
// might have just been put in a block
if (res.txs[n].in_pool)
{
cryptonote::transaction tx;
cryptonote::blobdata bd;
crypto::hash tx_hash, tx_prefix_hash;
if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[n].as_hex, bd))
{
if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash))
{
if (tx_hash == txid)
{
process_new_transaction(txid, tx, std::vector<uint64_t>(), 0, time(NULL), false, true);
m_scanned_pool_txs[0].insert(txid);
if (m_scanned_pool_txs[0].size() > 5000)
{
std::swap(m_scanned_pool_txs[0], m_scanned_pool_txs[1]);
m_scanned_pool_txs[0].clear();
}
}
else
{
LOG_PRINT_L0("Mismatched txids when processing unconfimed txes from pool");
}
}
else
{
LOG_PRINT_L0("failed to validate transaction from daemon");
}
}
else
{
LOG_PRINT_L0("Failed to parse tx " << txid);
}
}
else
{
LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more");
}
++n;
}
}
else
{
LOG_PRINT_L0("Expected " << txids.size() << " tx(es), got " << res.txs.size());
}
}
else
{
LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status);
}
}
MDEBUG("update_pool_state end");