diff --git a/.gitignore b/.gitignore index 3f5db06554ea..6f9de13ba1de 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ dash-cli dashd dash-qt make +wallet-utility diff --git a/Makefile.am b/Makefile.am index 816bd7f80b34..de5226177cce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,6 +12,7 @@ endif BITCOIND_BIN=$(top_builddir)/src/dashd$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/dash-qt$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/dash-cli$(EXEEXT) +WALLET_UTILITY_BIN=$(top_builddir)/src/wallet-utility$(EXEEXT) BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT) OSX_APP=Dash-Qt.app @@ -63,6 +64,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(WALLET_UTILITY_BIN) $(top_builddir)/release @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \ echo error: could not build $@ @echo built $@ @@ -145,6 +147,9 @@ $(BITCOIND_BIN): FORCE $(BITCOIN_CLI_BIN): FORCE $(MAKE) -C src $(@F) +$(WALLET_UTILITY_BIN): FORCE + $(MAKE) -C src $(@F) + if USE_LCOV baseline.info: diff --git a/configure.ac b/configure.ac index 333dbf8eebfb..babe795de411 100644 --- a/configure.ac +++ b/configure.ac @@ -191,7 +191,7 @@ CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], - [build dash-cli dash-tx (default=yes)])], + [build dash-cli dash-tx wallet-utility (default=yes)])], [build_bitcoin_utils=$withval], [build_bitcoin_utils=yes]) @@ -781,7 +781,7 @@ AC_MSG_CHECKING([whether to build dashd]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) -AC_MSG_CHECKING([whether to build utils (dash-cli dash-tx)]) +AC_MSG_CHECKING([whether to build utils (dash-cli dash-tx wallet-utility)]) AM_CONDITIONAL([BUILD_BITCOIN_UTILS], [test x$build_bitcoin_utils = xyes]) AC_MSG_RESULT($build_bitcoin_utils) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 5e745d54bbca..986599297831 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -171,7 +171,7 @@ def run_test(self): assert_equal(balance2["balance"], change_amount) # Check that deltas are returned correctly - deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 1, "end": 200}) balance3 = 0 for delta in deltas: balance3 += delta["satoshis"] @@ -323,6 +323,27 @@ def run_test(self): mempool_deltas = self.nodes[2].getaddressmempool({"addresses": [address1]}) assert_equal(len(mempool_deltas), 2) + # Include chaininfo in results + print "Testing results with chain info..." + + deltas_with_info = self.nodes[1].getaddressdeltas({ + "addresses": [address2], + "start": 1, + "end": 200, + "chainInfo": True + }) + start_block_hash = self.nodes[1].getblockhash(1); + end_block_hash = self.nodes[1].getblockhash(200); + assert_equal(deltas_with_info["start"]["height"], 1) + assert_equal(deltas_with_info["start"]["hash"], start_block_hash) + assert_equal(deltas_with_info["end"]["height"], 200) + assert_equal(deltas_with_info["end"]["hash"], end_block_hash) + + utxos_with_info = self.nodes[1].getaddressutxos({"addresses": [address2], "chainInfo": True}) + expected_tip_block_hash = self.nodes[1].getblockhash(267); + assert_equal(utxos_with_info["height"], 267) + assert_equal(utxos_with_info["hash"], expected_tip_block_hash) + print "Passed\n" diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 39f7ede05ab9..bd945ec6e9a7 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -104,7 +104,7 @@ def run_test(self): assert_equal(txVerbose3["vin"][0]["valueSat"], amount) # Check the database index - self.nodes[0].generate(1) + block_hash = self.nodes[0].generate(1) self.sync_all() txVerbose4 = self.nodes[3].getrawtransaction(txid2, 1) @@ -112,6 +112,26 @@ def run_test(self): assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"])) assert_equal(txVerbose4["vin"][0]["valueSat"], amount) + + # Check block deltas + print "Testing getblockdeltas..." + + block = self.nodes[3].getblockdeltas(block_hash[0]) + assert_equal(len(block["deltas"]), 2) + assert_equal(block["deltas"][0]["index"], 0) + assert_equal(len(block["deltas"][0]["inputs"]), 0) + assert_equal(len(block["deltas"][0]["outputs"]), 0) + assert_equal(block["deltas"][1]["index"], 1) + assert_equal(block["deltas"][1]["txid"], txid2) + assert_equal(block["deltas"][1]["inputs"][0]["index"], 0) + assert_equal(block["deltas"][1]["inputs"][0]["address"], "yeMpGzMj3rhtnz48XsfpB8itPHhHtgxLc3") + assert_equal(block["deltas"][1]["inputs"][0]["satoshis"], amount * -1) + assert_equal(block["deltas"][1]["inputs"][0]["prevtxid"], txid) + assert_equal(block["deltas"][1]["inputs"][0]["prevout"], 0) + assert_equal(block["deltas"][1]["outputs"][0]["index"], 0) + assert_equal(block["deltas"][1]["outputs"][0]["address"], "yeMpGzMj3rhtnz48XsfpB8itPHhHtgxLc3") + assert_equal(block["deltas"][1]["outputs"][0]["satoshis"], amount) + print "Passed\n" diff --git a/qa/rpc-tests/timestampindex.py b/qa/rpc-tests/timestampindex.py index 46ff710bdf37..289c81b2a521 100755 --- a/qa/rpc-tests/timestampindex.py +++ b/qa/rpc-tests/timestampindex.py @@ -35,15 +35,25 @@ def setup_network(self): self.sync_all() def run_test(self): - print "Mining 5 blocks..." - blockhashes = self.nodes[0].generate(5) - low = self.nodes[0].getblock(blockhashes[0])["time"] - high = self.nodes[0].getblock(blockhashes[4])["time"] + print "Mining 25 blocks..." + blockhashes = self.nodes[0].generate(25) + time.sleep(3) + print "Mining 25 blocks..." + blockhashes.extend(self.nodes[0].generate(25)) + time.sleep(3) + print "Mining 25 blocks..." + blockhashes.extend(self.nodes[0].generate(25)) self.sync_all() + low = self.nodes[1].getblock(blockhashes[0])["time"] + high = low + 76 + print "Checking timestamp index..." hashes = self.nodes[1].getblockhashes(high, low) - assert_equal(len(hashes), 5) - assert_equal(sorted(blockhashes), sorted(hashes)) + + assert_equal(len(hashes), len(blockhashes)) + + assert_equal(hashes, blockhashes) + print "Passed\n" diff --git a/src/Makefile.am b/src/Makefile.am index eba8d98c5155..5068166df75b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -60,6 +60,9 @@ endif if BUILD_BITCOIN_UTILS bin_PROGRAMS += dash-cli dash-tx +if ENABLE_WALLET + bin_PROGRAMS += wallet-utility +endif endif .PHONY: FORCE check-symbols check-security @@ -420,6 +423,14 @@ dash_cli_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) dash_cli_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) dash_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +# wallet-utility binary # +if ENABLE_WALLET +wallet_utility_SOURCES = wallet-utility.cpp +wallet_utility_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAG) +wallet_utility_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +wallet_utility_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +endif + if TARGET_WINDOWS dash_cli_SOURCES += dash-cli-res.rc endif @@ -429,6 +440,11 @@ dash_cli_LDADD = \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) dash_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) + +if ENABLE_WALLET +wallet_utility_LDADD = libbitcoin_wallet.a $(LIBBITCOIN_COMMON) $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1) $(LIBBITCOIN_UTIL) $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) +endif + # # dash-tx binary # diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d0e4c8139a7d..cd32910fe661 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -17,7 +17,9 @@ EXTRA_DIST += \ test/data/txcreate2.hex \ test/data/txcreatedata1.hex \ test/data/txcreatedata2.hex \ - test/data/txcreatesign.hex + test/data/txcreatesign.hex \ + test/wallet-utility.py \ + test/data/wallet.dat JSON_TEST_FILES = \ test/data/script_tests.json \ @@ -132,6 +134,10 @@ dash_test_clean : FORCE check-local: @echo "Running test/bitcoin-util-test.py..." $(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(srcdir)/test/bitcoin-util-test.py +if ENABLE_WALLET + @echo "Running test/wallet-utility.py..." + $(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(srcdir)/test/wallet-utility.py +endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 1907e2fa7843..5f1f175b836c 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -29,14 +29,14 @@ void HandleError(const leveldb::Status& status) throw(dbwrapper_error) throw dbwrapper_error("Unknown database error"); } -static leveldb::Options GetOptions(size_t nCacheSize) +static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles) { leveldb::Options options; options.block_cache = leveldb::NewLRUCache(nCacheSize / 2); options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously options.filter_policy = leveldb::NewBloomFilterPolicy(10); - options.compression = leveldb::kNoCompression; - options.max_open_files = 64; + options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression; + options.max_open_files = maxOpenFiles; if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error // on corruption in later versions. @@ -45,14 +45,14 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) +CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate, bool compression, int maxOpenFiles) { penv = NULL; readoptions.verify_checksums = true; iteroptions.verify_checksums = true; iteroptions.fill_cache = false; syncoptions.sync = true; - options = GetOptions(nCacheSize); + options = GetOptions(nCacheSize, compression, maxOpenFiles); options.create_if_missing = true; if (fMemory) { penv = leveldb::NewMemEnv(leveldb::Env::Default()); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5e7313f7eb54..2acd5fc05e07 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -169,14 +169,16 @@ class CDBWrapper public: /** - * @param[in] path Location in the filesystem where leveldb data will be stored. - * @param[in] nCacheSize Configures various leveldb cache settings. - * @param[in] fMemory If true, use leveldb's memory environment. - * @param[in] fWipe If true, remove all existing data. - * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR - * with a zero'd byte array. + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + * @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR + * with a zero'd byte array. + * @param[in] compression Enable snappy compression for the database + * @param[in] maxOpenFiles The maximum number of open files for the database */ - CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false); + CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false, bool compression = false, int maxOpenFiles = 64); ~CDBWrapper(); template diff --git a/src/init.cpp b/src/init.cpp index fa8037c34eb4..b3e7ddca4a2b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1445,18 +1445,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) } } + // block tree db settings + int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES); + bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION); + + LogPrintf("Block index database configuration:\n"); + LogPrintf("* Using %d max open files\n", dbMaxOpenFiles); + LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled"); + // cache size calculations int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache int64_t nBlockTreeDBCache = nTotalCache / 8; - if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) - nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) { + // enable 3/4 of the cache if addressindex and/or spentindex is enabled + nBlockTreeDBCache = nTotalCache * 3 / 4; + } else { + if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + } + } nTotalCache -= nBlockTreeDBCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nTotalCache -= nCoinDBCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache LogPrintf("Cache configuration:\n"); + LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024)); @@ -1477,7 +1492,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) delete pcoinscatcher; delete pblocktree; - pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); + pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles); pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); diff --git a/src/main.cpp b/src/main.cpp index b36c5d48e777..872b422056b4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1509,12 +1509,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } -bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, const bool fActiveOnly, std::vector > &hashes) { if (!fTimestampIndex) return error("Timestamp index not enabled"); - if (!pblocktree->ReadTimestampIndex(high, low, hashes)) + if (!pblocktree->ReadTimestampIndex(high, low, fActiveOnly, hashes)) return error("Unable to get hashes for timestamps"); return true; @@ -2831,10 +2831,27 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->UpdateSpentIndex(spentIndex)) return AbortNode(state, "Failed to write transaction index"); - if (fTimestampIndex) - if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + if (fTimestampIndex) { + unsigned int logicalTS = pindex->nTime; + unsigned int prevLogicalTS = 0; + + // retrieve logical timestamp of the previous block + if (pindex->pprev) + if (!pblocktree->ReadTimestampBlockIndex(pindex->pprev->GetBlockHash(), prevLogicalTS)) + LogPrintf("%s: Failed to read previous block's logical timestamp\n", __func__); + + if (logicalTS <= prevLogicalTS) { + logicalTS = prevLogicalTS + 1; + LogPrintf("%s: Previous logical timestamp is newer Actual[%d] prevLogical[%d] Logical[%d]\n", __func__, pindex->nTime, prevLogicalTS, logicalTS); + } + + if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(logicalTS, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); + if (!pblocktree->WriteTimestampBlockIndex(CTimestampBlockIndexKey(pindex->GetBlockHash()), CTimestampBlockIndexValue(logicalTS))) + return AbortNode(state, "Failed to write blockhash index"); + } + // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); diff --git a/src/main.h b/src/main.h index 28cc6497782f..8bc1098c2084 100644 --- a/src/main.h +++ b/src/main.h @@ -119,6 +119,8 @@ static const bool DEFAULT_TXINDEX = true; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; static const bool DEFAULT_SPENTINDEX = false; +static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000; +static const bool DEFAULT_DB_COMPRESSION = true; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -370,6 +372,65 @@ struct CTimestampIndexKey { } }; +struct CTimestampBlockIndexKey { + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 32; + } + + template + void Serialize(Stream& s, int nType, int nVersion) const { + blockHash.Serialize(s, nType, nVersion); + } + + template + void Unserialize(Stream& s, int nType, int nVersion) { + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampBlockIndexKey(uint256 hash) { + blockHash = hash; + } + + CTimestampBlockIndexKey() { + SetNull(); + } + + void SetNull() { + blockHash.SetNull(); + } +}; + +struct CTimestampBlockIndexValue { + unsigned int ltimestamp; + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, ltimestamp); + } + + template + void Unserialize(Stream& s, int nType, int nVersion) { + ltimestamp = ser_readdata32be(s); + } + + CTimestampBlockIndexValue (unsigned int time) { + ltimestamp = time; + } + + CTimestampBlockIndexValue() { + SetNull(); + } + + void SetNull() { + ltimestamp = 0; + } +}; + struct CAddressUnspentKey { unsigned int type; uint160 hashBytes; @@ -712,7 +773,7 @@ class CScriptCheck ScriptError GetScriptError() const { return error; } }; -bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, const bool fActiveOnly, std::vector > &hashes); bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index dc3141f7f53a..07d2c09a834e 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -5,6 +5,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "amount.h" +#include "base58.h" #include "chain.h" #include "chainparams.h" #include "checkpoints.h" @@ -14,6 +15,10 @@ #include "policy/policy.h" #include "primitives/transaction.h" #include "rpcserver.h" +#include "script/script.h" +#include "script/script_error.h" +#include "script/sign.h" +#include "script/standard.h" #include "streams.h" #include "sync.h" #include "txmempool.h" @@ -87,6 +92,112 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) return result; } +UniValue blockToDeltasJSON(const CBlock& block, const CBlockIndex* blockindex) +{ + UniValue result(UniValue::VOBJ); + result.push_back(Pair("hash", block.GetHash().GetHex())); + int confirmations = -1; + // Only report confirmations if the block is on the main chain + if (chainActive.Contains(blockindex)) { + confirmations = chainActive.Height() - blockindex->nHeight + 1; + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block is an orphan"); + } + result.push_back(Pair("confirmations", confirmations)); + result.push_back(Pair("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION))); + result.push_back(Pair("height", blockindex->nHeight)); + result.push_back(Pair("version", block.nVersion)); + result.push_back(Pair("merkleroot", block.hashMerkleRoot.GetHex())); + + UniValue deltas(UniValue::VARR); + + for (unsigned int i = 0; i < block.vtx.size(); i++) { + const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); + + UniValue entry(UniValue::VOBJ); + entry.push_back(Pair("txid", txhash.GetHex())); + entry.push_back(Pair("index", (int)i)); + + UniValue inputs(UniValue::VARR); + + if (!tx.IsCoinBase()) { + + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + + UniValue delta(UniValue::VOBJ); + + CSpentIndexValue spentInfo; + CSpentIndexKey spentKey(input.prevout.hash, input.prevout.n); + + if (GetSpentIndex(spentKey, spentInfo)) { + if (spentInfo.addressType == 1) { + delta.push_back(Pair("address", CBitcoinAddress(CKeyID(spentInfo.addressHash)).ToString())); + } else if (spentInfo.addressType == 2) { + delta.push_back(Pair("address", CBitcoinAddress(CScriptID(spentInfo.addressHash)).ToString())); + } else { + continue; + } + delta.push_back(Pair("satoshis", -1 * spentInfo.satoshis)); + delta.push_back(Pair("index", (int)j)); + delta.push_back(Pair("prevtxid", input.prevout.hash.GetHex())); + delta.push_back(Pair("prevout", (int)input.prevout.n)); + + inputs.push_back(delta); + } else { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Spent information not available"); + } + + } + } + + entry.push_back(Pair("inputs", inputs)); + + UniValue outputs(UniValue::VARR); + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + UniValue delta(UniValue::VOBJ); + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + delta.push_back(Pair("address", CBitcoinAddress(CScriptID(uint160(hashBytes))).ToString())); + + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + delta.push_back(Pair("address", CBitcoinAddress(CKeyID(uint160(hashBytes))).ToString())); + } else { + continue; + } + + delta.push_back(Pair("satoshis", out.nValue)); + delta.push_back(Pair("index", (int)k)); + + outputs.push_back(delta); + } + + entry.push_back(Pair("outputs", outputs)); + deltas.push_back(entry); + + } + result.push_back(Pair("deltas", deltas)); + result.push_back(Pair("time", block.GetBlockTime())); + result.push_back(Pair("mediantime", (int64_t)blockindex->GetMedianTimePast())); + result.push_back(Pair("nonce", (uint64_t)block.nNonce)); + result.push_back(Pair("bits", strprintf("%08x", block.nBits))); + result.push_back(Pair("difficulty", GetDifficulty(blockindex))); + result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex())); + + if (blockindex->pprev) + result.push_back(Pair("previousblockhash", blockindex->pprev->GetBlockHash().GetHex())); + CBlockIndex *pnext = chainActive.Next(blockindex); + if (pnext) + result.push_back(Pair("nextblockhash", pnext->GetBlockHash().GetHex())); + return result; +} + UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false) { UniValue result(UniValue::VOBJ); @@ -276,35 +387,97 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getblockdeltas(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error(""); + + std::string strHash = params[0].get_str(); + uint256 hash(uint256S(strHash)); + + if (mapBlockIndex.count(hash) == 0) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + + CBlock block; + CBlockIndex* pblockindex = mapBlockIndex[hash]; + + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) + throw JSONRPCError(RPC_INTERNAL_ERROR, "Block not available (pruned data)"); + + if(!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) + throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); + + return blockToDeltasJSON(block, pblockindex); +} + UniValue getblockhashes(const UniValue& params, bool fHelp) { - if (fHelp || params.size() != 2) + if (fHelp || params.size() < 2) throw runtime_error( "getblockhashes timestamp\n" "\nReturns array of hashes of blocks within the timestamp range provided.\n" "\nArguments:\n" "1. high (numeric, required) The newer block timestamp\n" "2. low (numeric, required) The older block timestamp\n" + "3. options (string, required) A json object\n" + " {\n" + " \"noOrphans\":true (boolean) will only include blocks on the main chain\n" + " \"logicalTimes\":true (boolean) will include logical timestamps with hashes\n" + " }\n" "\nResult:\n" "[\n" " \"hash\" (string) The block hash\n" "]\n" + "[\n" + " {\n" + " \"blockhash\": (string) The block hash\n" + " \"logicalts\": (numeric) The logical timestamp\n" + " }\n" + "]\n" "\nExamples:\n" + HelpExampleCli("getblockhashes", "1231614698 1231024505") + HelpExampleRpc("getblockhashes", "1231614698, 1231024505") - ); + + HelpExampleCli("getblockhashes", "1231614698 1231024505 '{\"noOrphans\":false, \"logicalTimes\":true}'") + ); unsigned int high = params[0].get_int(); unsigned int low = params[1].get_int(); - std::vector blockHashes; + bool fActiveOnly = false; + bool fLogicalTS = false; - if (!GetTimestampIndex(high, low, blockHashes)) { + if (params.size() > 2) { + if (params[2].isObject()) { + UniValue noOrphans = find_value(params[2].get_obj(), "noOrphans"); + UniValue returnLogical = find_value(params[2].get_obj(), "logicalTimes"); + + if (noOrphans.isBool()) + fActiveOnly = noOrphans.get_bool(); + + if (returnLogical.isBool()) + fLogicalTS = returnLogical.get_bool(); + } + } + + std::vector > blockHashes; + + if (fActiveOnly) + LOCK(cs_main); + + if (!GetTimestampIndex(high, low, fActiveOnly, blockHashes)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); } UniValue result(UniValue::VARR); - for (std::vector::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { - result.push_back(it->GetHex()); + + for (std::vector >::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { + if (fLogicalTS) { + UniValue item(UniValue::VOBJ); + item.push_back(Pair("blockhash", it->first.GetHex())); + item.push_back(Pair("logicalts", (int)it->second)); + result.push_back(item); + } else { + result.push_back(it->first.GetHex()); + } } return result; diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index e10aa4886160..580053f262a8 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -118,6 +118,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "voteraw", 4 }, { "getblockhashes", 0 }, { "getblockhashes", 1 }, + { "getblockhashes", 2 }, { "getspentinfo", 0}, { "getaddresstxids", 0}, { "getaddressbalance", 0}, diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 21de7ccfb9f3..305f90830819 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -652,7 +652,8 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) " [\n" " \"address\" (string) The base58check encoded address\n" " ,...\n" - " ]\n" + " ],\n" + " \"chainInfo\" (boolean) Include chain info with results\n" "}\n" "\nResult\n" "[\n" @@ -670,6 +671,14 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) + HelpExampleRpc("getaddressutxos", "{\"addresses\": [\"XwnLY9Tf7Zsef8gMGL2fhWA9ZmMjt4KPwg\"]}") ); + bool includeChainInfo = false; + if (params[0].isObject()) { + UniValue chainInfo = find_value(params[0].get_obj(), "chainInfo"); + if (chainInfo.isBool()) { + includeChainInfo = chainInfo.get_bool(); + } + } + std::vector > addresses; if (!getAddressesFromParams(params, addresses)) { @@ -686,7 +695,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) std::sort(unspentOutputs.begin(), unspentOutputs.end(), heightSort); - UniValue result(UniValue::VARR); + UniValue utxos(UniValue::VARR); for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); @@ -701,10 +710,20 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); output.push_back(Pair("height", it->second.blockHeight)); - result.push_back(output); + utxos.push_back(output); } - return result; + if (includeChainInfo) { + UniValue result(UniValue::VOBJ); + result.push_back(Pair("utxos", utxos)); + + LOCK(cs_main); + result.push_back(Pair("hash", chainActive.Tip()->GetBlockHash().GetHex())); + result.push_back(Pair("height", (int)chainActive.Height())); + return result; + } else { + return utxos; + } } UniValue getaddressdeltas(const UniValue& params, bool fHelp) @@ -722,6 +741,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) " ]\n" " \"start\" (number) The start block height\n" " \"end\" (number) The end block height\n" + " \"chainInfo\" (boolean) Include chain info in results, only applies if start and end specified\n" "}\n" "\nResult:\n" "[\n" @@ -742,12 +762,21 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue startValue = find_value(params[0].get_obj(), "start"); UniValue endValue = find_value(params[0].get_obj(), "end"); + UniValue chainInfo = find_value(params[0].get_obj(), "chainInfo"); + bool includeChainInfo = false; + if (chainInfo.isBool()) { + includeChainInfo = chainInfo.get_bool(); + } + int start = 0; int end = 0; if (startValue.isNum() && endValue.isNum()) { start = startValue.get_int(); end = endValue.get_int(); + if (start <= 0 || end <= 0) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Start and end is expected to be greater than zero"); + } if (end < start) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "End value is expected to be greater than start"); } @@ -773,7 +802,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) } } - UniValue result(UniValue::VARR); + UniValue deltas(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { std::string address; @@ -788,10 +817,38 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) delta.push_back(Pair("blockindex", (int)it->first.txindex)); delta.push_back(Pair("height", it->first.blockHeight)); delta.push_back(Pair("address", address)); - result.push_back(delta); + deltas.push_back(delta); } - return result; + UniValue result(UniValue::VOBJ); + + if (includeChainInfo && start > 0 && end > 0) { + LOCK(cs_main); + + if (start > chainActive.Height() || end > chainActive.Height()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Start or end is outside chain range"); + } + + CBlockIndex* startIndex = chainActive[start]; + CBlockIndex* endIndex = chainActive[end]; + + UniValue startInfo(UniValue::VOBJ); + UniValue endInfo(UniValue::VOBJ); + + startInfo.push_back(Pair("hash", startIndex->GetBlockHash().GetHex())); + startInfo.push_back(Pair("height", start)); + + endInfo.push_back(Pair("hash", endIndex->GetBlockHash().GetHex())); + endInfo.push_back(Pair("height", end)); + + result.push_back(Pair("deltas", deltas)); + result.push_back(Pair("start", startInfo)); + result.push_back(Pair("end", endInfo)); + + return result; + } else { + return deltas; + } } UniValue getaddressbalance(const UniValue& params, bool fHelp) diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index f9573800ec70..5474749007e7 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -60,8 +60,10 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud out.push_back(Pair("addresses", a)); } -void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) +void TxToJSONExpanded(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, + int nHeight = 0, int nConfirmations = 0, int nBlockTime = 0) { + uint256 txid = tx.GetHash(); entry.push_back(Pair("txid", txid.GetHex())); entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); @@ -122,6 +124,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) } entry.push_back(Pair("vout", vout)); + if (!hashBlock.IsNull()) { + entry.push_back(Pair("blockhash", hashBlock.GetHex())); + + if (nConfirmations > 0) { + entry.push_back(Pair("height", nHeight)); + entry.push_back(Pair("confirmations", nConfirmations)); + entry.push_back(Pair("time", nBlockTime)); + entry.push_back(Pair("blocktime", nBlockTime)); + } else { + entry.push_back(Pair("height", -1)); + entry.push_back(Pair("confirmations", 0)); + } + } + +} + +void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) +{ + + uint256 txid = tx.GetHash(); + entry.push_back(Pair("txid", txid.GetHex())); + entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); + entry.push_back(Pair("version", tx.nVersion)); + entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); + + UniValue vin(UniValue::VARR); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + UniValue in(UniValue::VOBJ); + if (tx.IsCoinBase()) + in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); + else { + in.push_back(Pair("txid", txin.prevout.hash.GetHex())); + in.push_back(Pair("vout", (int64_t)txin.prevout.n)); + UniValue o(UniValue::VOBJ); + o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); + o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); + in.push_back(Pair("scriptSig", o)); + } + in.push_back(Pair("sequence", (int64_t)txin.nSequence)); + vin.push_back(in); + } + entry.push_back(Pair("vin", vin)); + + UniValue vout(UniValue::VARR); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + const CTxOut& txout = tx.vout[i]; + UniValue out(UniValue::VOBJ); + out.push_back(Pair("value", ValueFromAmount(txout.nValue))); + out.push_back(Pair("valueSat", txout.nValue)); + out.push_back(Pair("n", (int64_t)i)); + UniValue o(UniValue::VOBJ); + ScriptPubKeyToJSON(txout.scriptPubKey, o, true); + out.push_back(Pair("scriptPubKey", o)); + vout.push_back(out); + } + entry.push_back(Pair("vout", vout)); + if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); BlockMap::iterator mi = mapBlockIndex.find(hashBlock); @@ -207,8 +266,6 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) + HelpExampleRpc("getrawtransaction", "\"mytxid\", 1") ); - LOCK(cs_main); - uint256 hash = ParseHashV(params[0], "parameter 1"); bool fVerbose = false; @@ -216,9 +273,31 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) fVerbose = (params[1].get_int() != 0); CTransaction tx; + uint256 hashBlock; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + int nHeight = 0; + int nConfirmations = 0; + int nBlockTime = 0; + + { + LOCK(cs_main); + if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + + BlockMap::iterator mi = mapBlockIndex.find(hashBlock); + if (mi != mapBlockIndex.end() && (*mi).second) { + CBlockIndex* pindex = (*mi).second; + if (chainActive.Contains(pindex)) { + nHeight = pindex->nHeight; + nConfirmations = 1 + chainActive.Height() - pindex->nHeight; + nBlockTime = pindex->GetBlockTime(); + } else { + nHeight = -1; + nConfirmations = 0; + nBlockTime = pindex->GetBlockTime(); + } + } + } string strHex = EncodeHexTx(tx); @@ -227,7 +306,8 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", strHex)); - TxToJSON(tx, hashBlock, result); + TxToJSONExpanded(tx, hashBlock, result, nHeight, nConfirmations, nBlockTime); + return result; } diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 343206e8e372..8762b9860f52 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -280,6 +280,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "getbestblockhash", &getbestblockhash, true }, { "blockchain", "getblockcount", &getblockcount, true }, { "blockchain", "getblock", &getblock, true }, + { "blockchain", "getblockdeltas", &getblockdeltas, false }, { "blockchain", "getblockhashes", &getblockhashes, true }, { "blockchain", "getblockhash", &getblockhash, true }, { "blockchain", "getblockheader", &getblockheader, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index b9e22aba22b0..47544a07574f 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -283,6 +283,7 @@ extern UniValue settxfee(const UniValue& params, bool fHelp); extern UniValue getmempoolinfo(const UniValue& params, bool fHelp); extern UniValue getrawmempool(const UniValue& params, bool fHelp); extern UniValue getblockhashes(const UniValue& params, bool fHelp); +extern UniValue getblockdeltas(const UniValue& params, bool fHelp); extern UniValue getblockhash(const UniValue& params, bool fHelp); extern UniValue getblockheader(const UniValue& params, bool fHelp); extern UniValue getblockheaders(const UniValue& params, bool fHelp); diff --git a/src/test/data/wallet.dat b/src/test/data/wallet.dat new file mode 100644 index 000000000000..105d8923e800 Binary files /dev/null and b/src/test/data/wallet.dat differ diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 4b089d32e36b..40f1234cdf88 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -47,6 +47,49 @@ BOOST_AUTO_TEST_CASE(dbwrapper) } } +BOOST_AUTO_TEST_CASE(dbwrapper_compression) +{ + // Perform tests both with compression and without + for (int i = 0; i < 2; i++) { + bool compression = (bool)i; + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, compression); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_64) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 64); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); +} + +BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_1000) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 1000); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); +} + // Test batch operations BOOST_AUTO_TEST_CASE(dbwrapper_batch) { diff --git a/src/test/wallet-utility.py b/src/test/wallet-utility.py new file mode 100755 index 000000000000..50e11ecd79dc --- /dev/null +++ b/src/test/wallet-utility.py @@ -0,0 +1,61 @@ +#!/usr/bin/python +# Copyright 2014 BitPay, Inc. +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import subprocess +import os +import json +import sys +import buildenv +import shutil + +def assert_equal(thing1, thing2): + if thing1 != thing2: + raise AssertionError("%s != %s"%(str(thing1),str(thing2))) + +if __name__ == '__main__': + datadir = os.environ["srcdir"] + "/test/data" + execprog = './wallet-utility' + buildenv.exeext + execargs = '-datadir=' + datadir + execrun = execprog + ' ' + execargs + + proc = subprocess.Popen(execrun, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) + try: + outs = proc.communicate() + except OSError: + print("OSError, Failed to execute " + execprog) + sys.exit(1) + + output = json.loads(outs[0]) + + assert_equal(output[0], "XwKHjUS2pS4inZT8nkqDSybMHi5hCYNSNx") + assert_equal(output[1], "XrihbS2Y29C3Gca72UWJiarUeaCHu7wumB") + assert_equal(output[2], "XqWwRpETvism6uSiAv2oHj4gxzUvtxSNkW") + + execargs = '-datadir=' + datadir + ' -dumppass' + execrun = execprog + ' ' + execargs + + proc = subprocess.Popen(execrun, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) + try: + outs = proc.communicate() + except OSError: + print("OSError, Failed to execute " + execprog) + sys.exit(1) + + output = json.loads(outs[0]) + + assert_equal(output[0]['addr'], "XwKHjUS2pS4inZT8nkqDSybMHi5hCYNSNx") + assert_equal(output[0]['pkey'], "XKk4fT2aB1pW5sGa6pBF8MA4bUaAcVEpWUnZQWzDc2XKUBkvxDq1") + assert_equal(output[1]['addr'], "XrihbS2Y29C3Gca72UWJiarUeaCHu7wumB") + assert_equal(output[1]['pkey'], "XBMge2M9Fd9YvCw3JqSE1aWEw56xJeRxRcQvb3JztsR45yK12s1H") + assert_equal(output[2]['addr'], "XqWwRpETvism6uSiAv2oHj4gxzUvtxSNkW") + assert_equal(output[2]['pkey'], "XBL71uLhA2p8q3qtXRSaLuQ5rwRb9gxFAeuhSbofwNcr26bGexfG") + + if os.path.exists(datadir + '/database'): + if os.path.isdir(datadir + '/database'): + shutil.rmtree(datadir + '/database') + + if os.path.exists(datadir + '/db.log'): + os.remove(datadir + '/db.log') + sys.exit(0) diff --git a/src/txdb.cpp b/src/txdb.cpp index 48150198f3a3..08ef51ca12f3 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -24,6 +24,7 @@ static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; +static const char DB_BLOCKHASHINDEX = 'z'; static const char DB_SPENTINDEX = 'p'; static const char DB_BLOCK_INDEX = 'b'; @@ -33,7 +34,7 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true, false, 64) { } @@ -75,7 +76,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, false, compression, maxOpenFiles) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { @@ -275,7 +276,7 @@ bool CBlockTreeDB::WriteTimestampIndex(const CTimestampIndexKey ×tampIndex) return WriteBatch(batch); } -bool CBlockTreeDB::ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) { +bool CBlockTreeDB::ReadTimestampIndex(const unsigned int &high, const unsigned int &low, const bool fActiveOnly, std::vector > &hashes) { boost::scoped_ptr pcursor(NewIterator()); @@ -284,8 +285,15 @@ bool CBlockTreeDB::ReadTimestampIndex(const unsigned int &high, const unsigned i while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; - if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp <= high) { - hashes.push_back(key.second.blockHash); + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp < high) { + if (fActiveOnly) { + if (blockOnchainActive(key.second.blockHash)) { + hashes.push_back(std::make_pair(key.second.blockHash, key.second.timestamp)); + } + } else { + hashes.push_back(std::make_pair(key.second.blockHash, key.second.timestamp)); + } + pcursor->Next(); } else { break; @@ -295,6 +303,22 @@ bool CBlockTreeDB::ReadTimestampIndex(const unsigned int &high, const unsigned i return true; } +bool CBlockTreeDB::WriteTimestampBlockIndex(const CTimestampBlockIndexKey &blockhashIndex, const CTimestampBlockIndexValue &logicalts) { + CDBBatch batch(&GetObfuscateKey()); + batch.Write(make_pair(DB_BLOCKHASHINDEX, blockhashIndex), logicalts); + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadTimestampBlockIndex(const uint256 &hash, unsigned int <imestamp) { + + CTimestampBlockIndexValue(lts); + if (!Read(std::make_pair(DB_BLOCKHASHINDEX, hash), lts)) + return false; + + ltimestamp = lts.ltimestamp; + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } @@ -307,6 +331,16 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } +bool CBlockTreeDB::blockOnchainActive(const uint256 &hash) { + CBlockIndex* pblockindex = mapBlockIndex[hash]; + + if (!chainActive.Contains(pblockindex)) { + return false; + } + + return true; +} + bool CBlockTreeDB::LoadBlockIndexGuts() { boost::scoped_ptr pcursor(NewIterator()); diff --git a/src/txdb.h b/src/txdb.h index 14d501278f23..9545fb47fc37 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -24,6 +24,8 @@ struct CAddressIndexIteratorKey; struct CAddressIndexIteratorHeightKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; +struct CTimestampBlockIndexKey; +struct CTimestampBlockIndexValue; struct CSpentIndexKey; struct CSpentIndexValue; class uint256; @@ -54,7 +56,7 @@ class CCoinsViewDB : public CCoinsView class CBlockTreeDB : public CDBWrapper { public: - CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = true, int maxOpenFiles = 1000); private: CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&); @@ -77,10 +79,13 @@ class CBlockTreeDB : public CDBWrapper std::vector > &addressIndex, int start = 0, int end = 0); bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); - bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); + bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, const bool fActiveOnly, std::vector > &vect); + bool WriteTimestampBlockIndex(const CTimestampBlockIndexKey &blockhashIndex, const CTimestampBlockIndexValue &logicalts); + bool ReadTimestampBlockIndex(const uint256 &hash, unsigned int &logicalTS); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); + bool blockOnchainActive(const uint256 &hash); }; #endif // BITCOIN_TXDB_H diff --git a/src/wallet-utility.cpp b/src/wallet-utility.cpp new file mode 100644 index 000000000000..8771b67ed10f --- /dev/null +++ b/src/wallet-utility.cpp @@ -0,0 +1,339 @@ +#include +#include + +// Include local headers +#include "wallet/walletdb.h" +#include "util.h" +#include "base58.h" +#include "wallet/crypter.h" +#include + + +void show_help() +{ + std::cout << + "This program outputs Dash addresses and private keys from a wallet.dat file" << std::endl + << std::endl + << "Usage and options: " + << std::endl + << " -datadir= to tell the program where your wallet is" + << std::endl + << " -wallet= (Optional) if your wallet is not named wallet.dat" + << std::endl + << " -regtest or -testnet (Optional) dumps addresses from regtest/testnet" + << std::endl + << " -dumppass (Optional)if you want to extract private keys associated with addresses" + << std::endl + << " -pass= if you have encrypted private keys stored in your wallet" + << std::endl; +} + + +class WalletUtilityDB : public CDB +{ + private: + typedef std::map MasterKeyMap; + MasterKeyMap mapMasterKeys; + unsigned int nMasterKeyMaxID; + SecureString mPass; + std::vector vMKeys; + + public: + WalletUtilityDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnClose = true) : CDB(strFilename, pszMode, fFlushOnClose) + { + nMasterKeyMaxID = 0; + mPass.reserve(100); + } + + std::string getAddress(CDataStream ssKey); + std::string getKey(CDataStream ssKey, CDataStream ssValue); + std::string getCryptedKey(CDataStream ssKey, CDataStream ssValue, std::string masterPass); + bool updateMasterKeys(CDataStream ssKey, CDataStream ssValue); + bool parseKeys(bool dumppriv, std::string masterPass); + + bool DecryptSecret(const std::vector& vchCiphertext, const uint256& nIV, CKeyingMaterial& vchPlaintext); + bool Unlock(); + bool DecryptKey(const std::vector& vchCryptedSecret, const CPubKey& vchPubKey, CKey& key); +}; + + +/* + * Address from a public key in base58 + */ +std::string WalletUtilityDB::getAddress(CDataStream ssKey) +{ + CPubKey vchPubKey; + ssKey >> vchPubKey; + CKeyID id = vchPubKey.GetID(); + std::string strAddr = CBitcoinAddress(id).ToString(); + + return strAddr; +} + + +/* + * Non encrypted private key in WIF + */ +std::string WalletUtilityDB::getKey(CDataStream ssKey, CDataStream ssValue) +{ + std::string strKey; + CPubKey vchPubKey; + ssKey >> vchPubKey; + CPrivKey pkey; + CKey key; + + ssValue >> pkey; + if (key.Load(pkey, vchPubKey, true)) + strKey = CBitcoinSecret(key).ToString(); + + return strKey; +} + + +bool WalletUtilityDB::DecryptSecret(const std::vector& vchCiphertext, const uint256& nIV, CKeyingMaterial& vchPlaintext) +{ + CCrypter cKeyCrypter; + std::vector chIV(WALLET_CRYPTO_KEY_SIZE); + memcpy(&chIV[0], &nIV, WALLET_CRYPTO_KEY_SIZE); + + BOOST_FOREACH(const CKeyingMaterial vMKey, vMKeys) + { + if(!cKeyCrypter.SetKey(vMKey, chIV)) + continue; + if (cKeyCrypter.Decrypt(vchCiphertext, *((CKeyingMaterial*)&vchPlaintext))) + return true; + } + return false; +} + + +bool WalletUtilityDB::Unlock() +{ + CCrypter crypter; + CKeyingMaterial vMasterKey; + + BOOST_FOREACH(const MasterKeyMap::value_type& pMasterKey, mapMasterKeys) + { + if(!crypter.SetKeyFromPassphrase(mPass, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod)) + return false; + if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, vMasterKey)) + continue; // try another master key + vMKeys.push_back(vMasterKey); + } + return true; +} + + +bool WalletUtilityDB::DecryptKey(const std::vector& vchCryptedSecret, const CPubKey& vchPubKey, CKey& key) +{ + CKeyingMaterial vchSecret; + if(!DecryptSecret(vchCryptedSecret, vchPubKey.GetHash(), vchSecret)) + return false; + + if (vchSecret.size() != 32) + return false; + + key.Set(vchSecret.begin(), vchSecret.end(), vchPubKey.IsCompressed()); + return true; +} + + +/* + * Encrypted private key in WIF format + */ +std::string WalletUtilityDB::getCryptedKey(CDataStream ssKey, CDataStream ssValue, std::string masterPass) +{ + mPass = masterPass.c_str(); + CPubKey vchPubKey; + ssKey >> vchPubKey; + CKey key; + + std::vector vKey; + ssValue >> vKey; + + if (!Unlock()) + return ""; + + if(!DecryptKey(vKey, vchPubKey, key)) + return ""; + + std::string strKey = CBitcoinSecret(key).ToString(); + return strKey; +} + + +/* + * Master key derivation + */ +bool WalletUtilityDB::updateMasterKeys(CDataStream ssKey, CDataStream ssValue) +{ + unsigned int nID; + ssKey >> nID; + CMasterKey kMasterKey; + ssValue >> kMasterKey; + if (mapMasterKeys.count(nID) != 0) + { + std::cout << "Error reading wallet database: duplicate CMasterKey id " << nID << std::endl; + return false; + } + mapMasterKeys[nID] = kMasterKey; + + if (nMasterKeyMaxID < nID) + nMasterKeyMaxID = nID; + + return true; +} + + +/* + * Look at all the records and parse keys for addresses and private keys + */ +bool WalletUtilityDB::parseKeys(bool dumppriv, std::string masterPass) +{ + DBErrors result = DB_LOAD_OK; + std::string strType; + bool first = true; + + try { + Dbc* pcursor = GetCursor(); + if (!pcursor) + { + LogPrintf("Error getting wallet database cursor\n"); + result = DB_CORRUPT; + } + + if (dumppriv) + { + while (result == DB_LOAD_OK && true) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + int result = ReadAtCursor(pcursor, ssKey, ssValue); + + if (result == DB_NOTFOUND) { + break; + } + else if (result != 0) + { + LogPrintf("Error reading next record from wallet database\n"); + result = DB_CORRUPT; + break; + } + + ssKey >> strType; + if (strType == "mkey") + { + updateMasterKeys(ssKey, ssValue); + } + } + pcursor->close(); + pcursor = GetCursor(); + } + + while (result == DB_LOAD_OK && true) + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + int ret = ReadAtCursor(pcursor, ssKey, ssValue); + + if (ret == DB_NOTFOUND) + { + std::cout << " ]" << std::endl; + first = true; + break; + } + else if (ret != DB_LOAD_OK) + { + LogPrintf("Error reading next record from wallet database\n"); + result = DB_CORRUPT; + break; + } + + ssKey >> strType; + + if (strType == "key" || strType == "ckey") + { + std::string strAddr = getAddress(ssKey); + std::string strKey = ""; + + + if (dumppriv && strType == "key") + strKey = getKey(ssKey, ssValue); + if (dumppriv && strType == "ckey") + { + if (masterPass == "") + { + std::cout << "Encrypted wallet, please provide a password. See help below" << std::endl; + show_help(); + result = DB_LOAD_FAIL; + break; + } + strKey = getCryptedKey(ssKey, ssValue, masterPass); + } + + if (strAddr != "") + { + if (first) + std::cout << "[ "; + else + std::cout << ", "; + } + + if (dumppriv) + { + std::cout << "{\"addr\" : \"" + strAddr + "\", " + << "\"pkey\" : \"" + strKey + "\"}" + << std::flush; + } + else + { + std::cout << "\"" + strAddr + "\""; + } + + first = false; + } + } + + pcursor->close(); + } catch (DbException &e) { + std::cout << "DBException caught " << e.get_errno() << std::endl; + } catch (std::exception &e) { + std::cout << "Exception caught " << std::endl; + } + + if (result == DB_LOAD_OK) + return true; + else + return false; +} + + +int main(int argc, char* argv[]) +{ + ParseParameters(argc, argv); + std::string walletFile = GetArg("-wallet", "wallet.dat"); + std::string masterPass = GetArg("-pass", ""); + bool fDumpPass = GetBoolArg("-dumppass", false); + bool help = GetBoolArg("-h", false); + bool result = false; + + if (help) + { + show_help(); + return 0; + } + + try { + SelectParams(ChainNameFromCommandLine()); + result = WalletUtilityDB(walletFile, "r").parseKeys(fDumpPass, masterPass); + } + catch (const std::exception& e) { + std::cout << "Error opening wallet file " << walletFile << std::endl; + std::cout << e.what() << std::endl; + } + + if (result) + return 0; + else + return -1; +}