From 5ec9b3f8ed08d2d7c584c7a8941c2c21e0a5f63e Mon Sep 17 00:00:00 2001 From: goatpig Date: Sat, 5 Sep 2026 09:30:29 +0200 Subject: [PATCH 01/26] improve blockxor test, exact size in FileCopy --- cppForSwig/Utils/FileUtils.cpp | 20 +++++------ cppForSwig/gtest/CppBlockUtilsTests.cpp | 48 +++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/cppForSwig/Utils/FileUtils.cpp b/cppForSwig/Utils/FileUtils.cpp index fa069eef7..43da0794d 100644 --- a/cppForSwig/Utils/FileUtils.cpp +++ b/cppForSwig/Utils/FileUtils.cpp @@ -201,7 +201,7 @@ FileCopy::FileCopy(const std::filesystem::path& path, size_t offset) if (size == 0) { throw std::runtime_error("empty file"); } - _lseek(fd, offset_, SEEK_SET); + _lseek(fd, offset, SEEK_SET); #else auto flag = O_RDONLY; fd = open(path.c_str(), flag); @@ -213,24 +213,24 @@ FileCopy::FileCopy(const std::filesystem::path& path, size_t offset) if (size == 0) { throw std::runtime_error("empty file"); } - lseek(fd, offset_, SEEK_SET); + lseek(fd, offset, SEEK_SET); #endif - if (offset_ >= size) { + if (offset >= size) { throw std::runtime_error("offset is too large"); } - //8 align the buffer + //8 align the underlying buffer, resize to exact length size_t sizeCount = (size - offset + 7) / 8; - data_.resize(sizeCount * 8); + data_.reserve(sizeCount * 8); + data_.resize(size - offset); #ifdef _WIN32 - _read(fd, &data_[0], size-offset_); + _read(fd, &data_[0], size - offset); _close(fd); #else - read(fd, &data_[0], size-offset_); + read(fd, &data_[0], size - offset); close(fd); #endif - } catch (const std::runtime_error &e) { if (fd != 0) { #ifdef _WIN32 @@ -270,12 +270,12 @@ const uint8_t* FileCopy::ptr() const void FileCopy::xorMe(uint64_t xorKey) { - if (data_.size() % 8 != 0) { + if (data_.capacity() % 8 != 0) { throw std::length_error("xored block data is misaligned"); } auto data64 = (uint64_t*)&data_[0]; - for (unsigned i = 0; i < data_.size() / 8; i++) { + for (unsigned i = 0; i < data_.capacity() / 8; i++) { data64[i] ^= xorKey; } } diff --git a/cppForSwig/gtest/CppBlockUtilsTests.cpp b/cppForSwig/gtest/CppBlockUtilsTests.cpp index 8f618e72d..6b26c97c8 100644 --- a/cppForSwig/gtest/CppBlockUtilsTests.cpp +++ b/cppForSwig/gtest/CppBlockUtilsTests.cpp @@ -1726,12 +1726,23 @@ TEST_F(BlockUtilsBare, BlockXor) } } - //swap the files + //remove clear blk file fileMap.close(); std::filesystem::remove(blk0dat_); - std::filesystem::rename(xoredFilePath, blk0dat_); - //create xor file, this is where the xorkey sits + //copy first 4 xored blocks + { + std::fstream blkFile; + blkFile.open(blk0dat_, std::ios::out | std::ios::binary); + + std::fstream xoredFile; + xoredFile.open(xoredFilePath, std::ios::in | std::ios::binary); + + std::copy_n(std::istreambuf_iterator(xoredFile), 3242, + std::ostreambuf_iterator(blkFile)); + } + + //create xor file, this is where the xor key sits { std::fstream xorFile; xorFile.open(blkdir_ / "blocks" / "xor.dat", std::ios::out | std::ios::binary); @@ -1777,10 +1788,41 @@ TEST_F(BlockUtilsBare, BlockXor) DBTestUtils::goOnline(clients_, bdvID); DBTestUtils::waitOnBDVReady(clients_, bdvID); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::SCRADDR), 3U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::SCRADDR), TestChain::blkHash3); + auto header = theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash3); + EXPECT_TRUE(header->isMainBranch()); + auto bdm = theBDMt_->bdm(); auto getBal = [bdm](const BinaryData& scrAddr)->uint64_t { return DBTestUtils::getScrAddrBalance(scrAddr, bdm); }; + //check balances + EXPECT_EQ(getBal(TestChain::scrAddrA), 50 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrB), 30 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrC), 55 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrD), 5 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrE), 30 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrF), 5 * COIN); + + EXPECT_EQ(getBal(TestChain::lb1ScrAddr), 10 * COIN); + EXPECT_EQ(getBal(TestChain::lb1ScrAddrP2SH), 0 * COIN); + EXPECT_EQ(getBal(TestChain::lb2ScrAddr), 10 * COIN); + EXPECT_EQ(getBal(TestChain::lb2ScrAddrP2SH), 5 * COIN); + + //add last 2 blocks + std::filesystem::remove(blk0dat_); + std::filesystem::rename(xoredFilePath, blk0dat_); + DBTestUtils::triggerNewBlockNotification(theBDMt_); + DBTestUtils::waitOnNewBlockSignal(clients_, bdvID); + + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::SCRADDR), 5U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::SCRADDR), TestChain::blkHash5); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash5)->isMainBranch()); + auto lastScannedRange = bdm->getLastScannedRange(); + EXPECT_EQ(lastScannedRange.first, TestChain::blkHash4); + EXPECT_EQ(lastScannedRange.second, TestChain::blkHash5); + //check balances EXPECT_EQ(getBal(TestChain::scrAddrA), 50 * COIN); EXPECT_EQ(getBal(TestChain::scrAddrB), 70 * COIN); From c39219b4d7022320938999e351df2bc4ea1282e2 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sat, 5 Sep 2026 20:41:35 +0200 Subject: [PATCH 02/26] txhints collision handler --- cppForSwig/BDM_Server.cpp | 34 +++++-- cppForSwig/BlockchainDatabase/Blockchain.cpp | 6 +- .../BlockchainDatabase/BlockchainData.cpp | 33 ++++++- .../BlockchainDatabase/BlockchainData.h | 2 + .../BlockchainDatabase/lmdb_wrapper.cpp | 7 +- cppForSwig/BlockchainDatabase/lmdb_wrapper.h | 2 +- cppForSwig/TxClasses.cpp | 17 ++++ cppForSwig/TxClasses.h | 16 ++++ cppForSwig/ZeroConf/Utils.cpp | 27 ++++-- cppForSwig/ZeroConf/Utils.h | 2 +- cppForSwig/gtest/CppBlockUtilsTests.cpp | 95 +++++++++++++++++++ cppForSwig/gtest/ZeroConfTests.cpp | 60 ++++++++++++ 12 files changed, 274 insertions(+), 27 deletions(-) diff --git a/cppForSwig/BDM_Server.cpp b/cppForSwig/BDM_Server.cpp index dd518aea9..797e9c747 100644 --- a/cppForSwig/BDM_Server.cpp +++ b/cppForSwig/BDM_Server.cpp @@ -151,6 +151,7 @@ namespace { case BdvRequest::Which::GET_TXS_BY_HASH: { auto db = bdv->getDB(); + auto blockData = bdv->bdm()->blockchainData(); auto txHashList = request.getGetTxsByHash(); std::map results; std::set possibleZcHashes; @@ -158,12 +159,18 @@ namespace { BinaryDataRef hashBdr(txHash.begin(), txHash.end()); try { auto txKey = db->getDBKeyForHash(hashBdr); + if (!Types::isTxKeyValid(txKey)) { + possibleZcHashes.emplace(hashBdr); + continue; + } auto tx = bdv->bdm()->blockchainData()->getTx(txKey); results.emplace(txKey, std::move(tx)); - } catch (const std::exception&) { + } catch (const TxHintCollision& collision) { //could not get the tx, maybe it's a zc? - possibleZcHashes.emplace(hashBdr); - continue; + auto txKey = blockData->resolveTxHintCollision(collision); + if (!Types::isTxKeyValid(txKey)) { + possibleZcHashes.emplace(hashBdr); + } } } @@ -1838,9 +1845,24 @@ void Clients::p2pBroadcast(Types::BdvId bdvId, std::vector& rawZC Tx tx(rawZcRef); auto hash = tx.getThisHash(); - auto dbKey = db->getDBKeyForHash(hash); - if (Types::isTxKeyValid(dbKey)) { - //notify the bdv of the error + Types::TxKey dbTxKey; + try { + //do we know this txhash? + dbTxKey = db->getDBKeyForHash(hash); + if (Types::isTxKeyValid(dbTxKey)) { + //we have a txkey for this hash, do a hard check against + //the actual tx data + std::set hints{dbTxKey}; + TxHintCollision collision{hash, hints}; + dbTxKey = bdm_->blockchainData()->resolveTxHintCollision(collision); + } + } catch (const TxHintCollision& collision){ + //multiple hints, do a hard check too + dbTxKey = bdm_->blockchainData()->resolveTxHintCollision(collision); + } + + if (Types::isTxKeyValid(dbTxKey)) { + //this hash is mined, notify the bdv of the error auto notifPacket = std::make_shared(); notifPacket->bdvPtr = BDVs_.get(bdvId); diff --git a/cppForSwig/BlockchainDatabase/Blockchain.cpp b/cppForSwig/BlockchainDatabase/Blockchain.cpp index a095d0f0a..06b505662 100644 --- a/cppForSwig/BlockchainDatabase/Blockchain.cpp +++ b/cppForSwig/BlockchainDatabase/Blockchain.cpp @@ -123,13 +123,11 @@ HeaderPtr Blockchain::getHeaderById(Types::BlockId id) const { std::unique_lock lock(mu_); if (id > highestBlockID_.load(std::memory_order_relaxed)) { - LOGERR << "block id " << id << " is too big"; - throw std::range_error("block id overflow"); + throw std::range_error(std::format("block id overflow: {}", id)); } auto header = headersById_[id]; if (header == nullptr) { - LOGERR << "cannot find block for id: " << id; - throw std::range_error("Cannot find block by id"); + throw std::range_error(std::format("cannot find block {} by id", id)); } return header; } diff --git a/cppForSwig/BlockchainDatabase/BlockchainData.cpp b/cppForSwig/BlockchainDatabase/BlockchainData.cpp index 7014aa5bc..3826f55e8 100644 --- a/cppForSwig/BlockchainDatabase/BlockchainData.cpp +++ b/cppForSwig/BlockchainDatabase/BlockchainData.cpp @@ -123,14 +123,41 @@ Hash32 BlockchainData::getTxHashForTxKey(const Types::TxKey& txKey) const return Hash32{tx.getThisHash()}; } +Types::TxKey BlockchainData::resolveTxHintCollision( + const TxHintCollision& collision) +{ + for (const auto& txKey : collision.getCandidates()) { + if (!isTxKeyOnMainBranch(txKey)) { + continue; + } + try { + auto txHash = getTxHashForTxKey(txKey); + if (txHash == collision.getTxHash()) { + return txKey; + } + } catch (const BlockchainDataException& e) { + //ignore mangled data + LOGWARN << "[getTxHashForTxKey] " << e.what(); + continue; + } catch (const std::range_error& e) { + LOGWARN << "[getTxHashForTxKey] " << e.what(); + } + } + return Types::INVALID_TX_KEY; +} + bool BlockchainData::isTxKeyOnMainBranch(const Types::TxKey& txKey) const { auto blockID = Types::getBlockIDFromTxKey(txKey); - auto header = blockchain_->getHeaderById(blockID); - if (header == nullptr) { + try { + auto header = blockchain_->getHeaderById(blockID); + if (header == nullptr) { + return false; + } + return header->isMainBranch(); + } catch (const std::range_error&) { return false; } - return header->isMainBranch(); } //////// diff --git a/cppForSwig/BlockchainDatabase/BlockchainData.h b/cppForSwig/BlockchainDatabase/BlockchainData.h index 56298dc17..943c9e9a6 100644 --- a/cppForSwig/BlockchainDatabase/BlockchainData.h +++ b/cppForSwig/BlockchainDatabase/BlockchainData.h @@ -15,6 +15,7 @@ #include class Tx; +class TxHintCollision; namespace Armory { @@ -42,6 +43,7 @@ namespace Armory Hash32 getTxHashForTxKey(const Types::TxKey&) const; bool isTxKeyOnMainBranch(const Types::TxKey&) const; + Types::TxKey resolveTxHintCollision(const TxHintCollision&); std::pair, size_t> getRawBlockForId( Types::BlockId) const; diff --git a/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp b/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp index ee9e1f5f0..8290bf2fe 100644 --- a/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp +++ b/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp @@ -649,7 +649,7 @@ Types::TxKey LMDBBlockDatabase::getDBKeyForHash( } } - //time to check tx hashes + //time to check tx hints auto hashTableIndex = txHash.getPtr()[8]; auto tx = beginHashTableTx(DB_SELECT::TXHINTS, hashTableIndex, LMDB::Mode::ReadOnly); @@ -691,12 +691,11 @@ Types::TxKey LMDBBlockDatabase::getDBKeyForHash( if (result.empty()) { return Types::INVALID_TX_KEY; } else if (result.size() == 1) { - //TODO: migrate to uint64_t txkeys return *result.begin(); } else { - //NOTE: db wrapper shouldnt have to pick the correct key, + //db wrapper shouldnt have to pick the correct key, //caller should deal with it - throw std::runtime_error("implement me"); + throw TxHintCollision(txHash, result); } } diff --git a/cppForSwig/BlockchainDatabase/lmdb_wrapper.h b/cppForSwig/BlockchainDatabase/lmdb_wrapper.h index dfe23c250..651ebcf18 100644 --- a/cppForSwig/BlockchainDatabase/lmdb_wrapper.h +++ b/cppForSwig/BlockchainDatabase/lmdb_wrapper.h @@ -216,7 +216,7 @@ class LMDBBlockDatabase DB_SELECT, uint8_t, LMDB::Mode) const; ///////////////////////////////////////////////////////////////////////////// - uint64_t getDBKeyForHash(const Armory::Types::TxHash&) const; + Armory::Types::TxKey getDBKeyForHash(const Armory::Types::TxHash&) const; ///////////////////////////////////////////////////////////////////////////// // Interface to translate Stored* objects to/from persistent DB storage diff --git a/cppForSwig/TxClasses.cpp b/cppForSwig/TxClasses.cpp index faaecdce9..af7b39141 100644 --- a/cppForSwig/TxClasses.cpp +++ b/cppForSwig/TxClasses.cpp @@ -48,6 +48,23 @@ uint64_t RecipientReuseException::value() const return value_; } +///////////////////////////////////////////////////////////////////////////// +// TxHintCollision +TxHintCollision::TxHintCollision( + const Types::TxHash& hash, std::set& hints) : + txHash_{hash}, candidates_{std::move(hints)} +{} + +const Types::TxHash& TxHintCollision::getTxHash() const +{ + return txHash_; +} + +const std::set& TxHintCollision::getCandidates() const +{ + return candidates_; +} + ///////////////////////////////////////////////////////////////////////////// // Outpoint Outpoint::Outpoint(const uint8_t* ptr, size_t remaining) diff --git a/cppForSwig/TxClasses.h b/cppForSwig/TxClasses.h index da64723d2..a859891da 100644 --- a/cppForSwig/TxClasses.h +++ b/cppForSwig/TxClasses.h @@ -14,6 +14,7 @@ #include #include +#include //PayStruct flags #define USE_FULL_CUSTOM_LIST 1 @@ -38,6 +39,21 @@ class RecipientReuseException uint64_t value(void) const; }; +////////////////////////////////////////////////////////////////////////// +// tx hint collision exception +class TxHintCollision +{ +private: + Armory::Types::TxHash txHash_; + std::set candidates_; + +public: + TxHintCollision(const Armory::Types::TxHash&, std::set&); + + const Armory::Types::TxHash& getTxHash(void) const; + const std::set& getCandidates(void) const; +}; + //////////////////////////////////////////////////////////////////////////////// // Outpoint is just a reference to a TxOut class Outpoint diff --git a/cppForSwig/ZeroConf/Utils.cpp b/cppForSwig/ZeroConf/Utils.cpp index f66adffa0..37aa3f680 100644 --- a/cppForSwig/ZeroConf/Utils.cpp +++ b/cppForSwig/ZeroConf/Utils.cpp @@ -178,8 +178,13 @@ void ZeroConf::preprocessTx(ParsedTx& tx, LMDBBlockDatabase* db, */ //sanity check: is this tx mined? - const auto& txHash = tx.getTxHash(); - auto txKey = db->getDBKeyForHash(txHash); + Types::TxKey txKey; + try { + const auto& txHash = tx.getTxHash(); + txKey = db->getDBKeyForHash(txHash); + } catch (const TxHintCollision& collision) { + txKey = bd->resolveTxHintCollision(collision); + } if (Types::isTxKeyValid(txKey)) { tx.state = ParsedTxStatus::Mined; return; @@ -224,7 +229,7 @@ void ZeroConf::preprocessTx(ParsedTx& tx, LMDBBlockDatabase* db, if (!opRef.isResolved()) { //resolve outpoint to dbkey - opRef.resolveDbKey(db); + opRef.resolveDbKey(db, bd); if (!opRef.isResolved()) { continue; } @@ -459,17 +464,23 @@ void OutPointRef::unserialize(BinaryDataRef bdr) unserialize(bdr.getPtr(), bdr.getSize()); } -void OutPointRef::resolveDbKey(LMDBBlockDatabase *dbPtr) +void OutPointRef::resolveDbKey( + LMDBBlockDatabase *dbPtr, std::shared_ptr bd) { if (txHash_.empty() || txOutIndex_ == UINT16_MAX) { throw std::runtime_error("empty outpoint hash"); } - auto key = dbPtr->getDBKeyForHash(txHash_); - if (key == Types::INVALID_TX_KEY) { - return; + Types::TxKey txKey; + try { + txKey = dbPtr->getDBKeyForHash(txHash_); + } catch (const TxHintCollision& collision) { + txKey = bd->resolveTxHintCollision(collision); + } + + if (Types::isTxKeyValid(txKey)) { + setDbKey(txKey); } - setDbKey(key); } bool OutPointRef::isResolved() const diff --git a/cppForSwig/ZeroConf/Utils.h b/cppForSwig/ZeroConf/Utils.h index ba4be4c4e..fa7092c7d 100644 --- a/cppForSwig/ZeroConf/Utils.h +++ b/cppForSwig/ZeroConf/Utils.h @@ -70,7 +70,7 @@ namespace Armory void unserialize(const uint8_t*, uint32_t); void unserialize(BinaryDataRef); - void resolveDbKey(LMDBBlockDatabase*); + void resolveDbKey(LMDBBlockDatabase*, std::shared_ptr); bool isResolved(void) const; bool isInitialized(void) const; diff --git a/cppForSwig/gtest/CppBlockUtilsTests.cpp b/cppForSwig/gtest/CppBlockUtilsTests.cpp index 6b26c97c8..089f10e9b 100644 --- a/cppForSwig/gtest/CppBlockUtilsTests.cpp +++ b/cppForSwig/gtest/CppBlockUtilsTests.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "BDM_mainthread.h" #include "Server.h" @@ -2106,6 +2107,100 @@ TEST_F(BlockUtilsFull, TxHints) keyPair = getTxKeyForHash(TestChain::hash52, db); EXPECT_EQ(keyPair.first, 5); EXPECT_EQ(keyPair.second, 2); + //check db returns hint for hash35 + auto txkey = db->getDBKeyForHash(TestChain::hash35); + ASSERT_EQ(txkey, 0xffff050004000000); + + //inject fake hints for hash35 + { + const uint64_t blockIDMask = 0x00000000FFFFFFFF; + uint8_t hashTableIndex = TestChain::hash35.getPtr()[8]; + auto tx = db->beginHashTableTx( + DB_SELECT::TXHINTS, hashTableIndex, LMDB::Mode::ReadWrite); + + //2.1, 2.4 (an non existant tx, to cover deser throws) + uint64_t blockID2 = 3ul << 32; + uint64_t txHintKey2; + std::memcpy(&txHintKey2, TestChain::hash35.getPtr(), 8); + txHintKey2 = (txHintKey2 & blockIDMask) | blockID2; + std::vector txids2{1, 4}; + tx->insert( + LMDB::DataRef{ + sizeof(uint64_t), + (const char*)&txHintKey2}, + LMDB::DataRef{ + sizeof(uint16_t) * txids2.size(), + (const char*)&txids2[0]} + ); + + //4.2, 4.4 + uint64_t blockID4 = 5ul << 32; + uint64_t txHintKey4; + std::memcpy(&txHintKey4, TestChain::hash35.getPtr(), 8); + txHintKey4 = (txHintKey4 & blockIDMask) | blockID4; + std::vector txids4{2, 4}; + tx->insert( + LMDB::DataRef{ + sizeof(uint64_t), + (const char*)&txHintKey4}, + LMDB::DataRef{ + sizeof(uint16_t) * txids4.size(), + (const char*)&txids4[0]} + ); + + //7.3 (inexistant block, to cover throws) + uint64_t blockID7 = 8ul << 32; + uint64_t txHintKey7; + std::memcpy(&txHintKey7, TestChain::hash35.getPtr(), 8); + txHintKey7 = (txHintKey7 & blockIDMask) | blockID7; + std::vector txids7{3}; + tx->insert( + LMDB::DataRef{ + sizeof(uint64_t), + (const char*)&txHintKey7}, + LMDB::DataRef{ + sizeof(uint16_t) * txids7.size(), + (const char*)&txids7[0]} + ); + } + + try { + db->getDBKeyForHash(TestChain::hash35); + ASSERT_TRUE(false); + } catch (const TxHintCollision& collision) { + ASSERT_EQ(collision.getTxHash(), TestChain::hash35); + + auto candidates = collision.getCandidates(); + ASSERT_EQ(candidates.size(), 6); + + //2.1 + auto candidatesIter = candidates.begin(); + ASSERT_EQ(*candidatesIter, 0xffff010003000000); + + //4.2 + ++candidatesIter; + ASSERT_EQ(*candidatesIter, 0xffff020005000000); + + //7.3 + ++candidatesIter; + ASSERT_EQ(*candidatesIter, 0xffff030008000000); + + //2.4 + ++candidatesIter; + ASSERT_EQ(*candidatesIter, 0xffff040003000000); + + //4.4 + ++candidatesIter; + ASSERT_EQ(*candidatesIter, 0xffff040005000000); + + //3.5 + ++candidatesIter; + ASSERT_EQ(*candidatesIter, 0xffff050004000000); + + auto final = bdm->blockchainData()->resolveTxHintCollision(collision); + ASSERT_EQ(final, 0xffff050004000000); + } + //cleanup bdvPtr.reset(); } diff --git a/cppForSwig/gtest/ZeroConfTests.cpp b/cppForSwig/gtest/ZeroConfTests.cpp index cff00ab0b..f0823d3f7 100644 --- a/cppForSwig/gtest/ZeroConfTests.cpp +++ b/cppForSwig/gtest/ZeroConfTests.cpp @@ -2092,6 +2092,66 @@ TEST_F(ZeroConfTests_FullNode, Replace_ZC_Test) ++utxoIter; } + //plot twist: insert fake txhint collision for first spender hash + { + auto firstTxHash = utxoVec[0].getTxHash(); + std::cout << firstTxHash.toHexStr() << std::endl; + ASSERT_EQ(firstTxHash, TestChain::hash32); + + //insert fake txhint: tx 4.1 + const uint64_t blockIDMask = 0x00000000FFFFFFFF; + uint8_t hashTableIndex = TestChain::hash32.getPtr()[8]; + auto db = bdm->getIFace(); + auto txkey = db->getDBKeyForHash(TestChain::hash32); + { + auto tx = db->beginHashTableTx( + DB_SELECT::TXHINTS, hashTableIndex, LMDB::Mode::ReadWrite); + + //2.1, 2.4 (an non existant tx, to cover deser throws) + uint64_t blockID = 5ul << 32; + uint64_t txHintKey; + std::memcpy(&txHintKey, TestChain::hash32.getPtr(), 8); + txHintKey = (txHintKey & blockIDMask) | blockID; + std::vector txids{1}; + tx->insert( + LMDB::DataRef{ + sizeof(uint64_t), + (const char*)&txHintKey}, + LMDB::DataRef{ + sizeof(uint16_t) * txids.size(), + (const char*)&txids[0]} + ); + } + + //for this test to work, have to get rid of TestChain::hash32 from the + //known hashes table to fall back to txhints + { + auto tx = db->beginTransaction( + DB_SELECT::KNOWNHASHES, LMDB::Mode::ReadWrite); + tx->erase(LMDB::DataRef{ + TestChain::hash32.getSize(), TestChain::hash32.getCharPtr()}); + } + + //check tx has 2 hints + try { + auto txkey = db->getDBKeyForHash(TestChain::hash32); + ASSERT_TRUE(false); + } catch (const TxHintCollision& collision) { + ASSERT_EQ(collision.getTxHash(), TestChain::hash32); + + auto candidates = collision.getCandidates(); + ASSERT_EQ(candidates.size(), 2); + + //4.1 + auto candidatesIter = candidates.begin(); + ASSERT_EQ(*candidatesIter, 0xffff010005000000); + + //3.2 + ++candidatesIter; + ASSERT_EQ(*candidatesIter, 0xffff020004000000); + } + } + //create script spender objects uint64_t total = 0; for (auto& utxo : utxoVec) { From 4db8d939829aa23537b9ebac2411af5eddd86b9c Mon Sep 17 00:00:00 2001 From: Tom Robson <47132116+tomrobbo@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:04:33 +1000 Subject: [PATCH 03/26] Rotate xor key by file offset in FileCopy::xorMe --- cppForSwig/Utils/FileUtils.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cppForSwig/Utils/FileUtils.cpp b/cppForSwig/Utils/FileUtils.cpp index 43da0794d..dcd96d3f5 100644 --- a/cppForSwig/Utils/FileUtils.cpp +++ b/cppForSwig/Utils/FileUtils.cpp @@ -274,6 +274,13 @@ void FileCopy::xorMe(uint64_t xorKey) throw std::length_error("xored block data is misaligned"); } + //Core's key repeats from the start of the file, so rotate it to + //match this copy's starting offset + unsigned shift = (offset_ % 8) * 8; + if (shift != 0) { + xorKey = (xorKey >> shift) | (xorKey << (64 - shift)); + } + auto data64 = (uint64_t*)&data_[0]; for (unsigned i = 0; i < data_.capacity() / 8; i++) { data64[i] ^= xorKey; From 59ccf9f52f74e82a10f7efb6416e076d3c2f1cf4 Mon Sep 17 00:00:00 2001 From: Tom Robson <47132116+tomrobbo@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:18:30 +1000 Subject: [PATCH 04/26] Update comment --- cppForSwig/Utils/FileUtils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cppForSwig/Utils/FileUtils.cpp b/cppForSwig/Utils/FileUtils.cpp index dcd96d3f5..ca6aef389 100644 --- a/cppForSwig/Utils/FileUtils.cpp +++ b/cppForSwig/Utils/FileUtils.cpp @@ -274,8 +274,8 @@ void FileCopy::xorMe(uint64_t xorKey) throw std::length_error("xored block data is misaligned"); } - //Core's key repeats from the start of the file, so rotate it to - //match this copy's starting offset + //the xor key is aligned to the start of the file, not the start of + //this copy, so rotate it to match offset_ unsigned shift = (offset_ % 8) * 8; if (shift != 0) { xorKey = (xorKey >> shift) | (xorKey << (64 - shift)); From 08a6acb07307d12e5fb273d40a130490e8fcbf13 Mon Sep 17 00:00:00 2001 From: Tom Robson <47132116+tomrobbo@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:18:52 +1000 Subject: [PATCH 05/26] Don't throw in canLegacySerialize when the answer is false --- armoryengine/CppBridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/armoryengine/CppBridge.py b/armoryengine/CppBridge.py index dd2c147af..620fd3dd3 100755 --- a/armoryengine/CppBridge.py +++ b/armoryengine/CppBridge.py @@ -1596,7 +1596,7 @@ def canLegacySerialize(self): packet.signer.canLegacySerialize = None fut = self.send(packet) - reply = fut.getVal() + reply = fut.getVal(nothrow=True) return reply.success ################################################################################ From 5ffd1075f63eb612e4dea6ffc2d7a84b8ea04238 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sun, 6 Sep 2026 16:24:46 +0200 Subject: [PATCH 06/26] fix key share sequence when automating db with manual core --- cppForSwig/Utils/BtcUtils.cpp | 3 ++- cppForSwig/main.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cppForSwig/Utils/BtcUtils.cpp b/cppForSwig/Utils/BtcUtils.cpp index c19809bc8..98fc73958 100644 --- a/cppForSwig/Utils/BtcUtils.cpp +++ b/cppForSwig/Utils/BtcUtils.cpp @@ -1457,7 +1457,8 @@ std::string BtcUtils::base64_decode(const std::string& in) if (val8 == '=' || val8 == '\n') { break; } - throw std::runtime_error("invalid b64 character"); + throw std::runtime_error( + std::format("invalid b64 character: {}", val8)); } uint32_t bits = iter->second << (26 - (6 * (y % 4))); diff --git a/cppForSwig/main.cpp b/cppForSwig/main.cpp index 44c9bbd0c..734ad5d81 100644 --- a/cppForSwig/main.cpp +++ b/cppForSwig/main.cpp @@ -59,9 +59,9 @@ int main(int argc, char* argv[]) } auto logFilePath = Config::Pathing::logFilePath(LOG_FILE_NAME).string(); + LOGDISABLESTDOUT(); STARTLOGGING(logFilePath, LogLvlDebug); if (Config::NetworkSettings::ephemeralPeers()) { - LOGDISABLESTDOUT(); if (Config::NetworkSettings::oneWayAuth()) { LOGERR << "--ephemeral and --oneWayAuth are mutually exclusive for db"; exit(-3); From 6432fd2b718faaab848d2854fd54f0963d194e83 Mon Sep 17 00:00:00 2001 From: goatpig Date: Mon, 7 Sep 2026 09:53:44 +0200 Subject: [PATCH 07/26] rescan test for DB_FULL, fix rescans for DB_FULL --- .../BlockchainDatabase/lmdb_wrapper.cpp | 6 +- cppForSwig/gtest/CppBlockUtilsTests.cpp | 106 ++++++++++++++++++ 2 files changed, 107 insertions(+), 5 deletions(-) diff --git a/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp b/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp index 8290bf2fe..1fd3a8964 100644 --- a/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp +++ b/cppForSwig/BlockchainDatabase/lmdb_wrapper.cpp @@ -58,7 +58,7 @@ extern const std::vector FULLNODEHASHTABLES{ extern const std::vector BARENODEDBS{ DB_SELECT::SCRADDR, DB_SELECT::TXOUTS, DB_SELECT::TXINS, - DB_SELECT::KNOWNHASHES, DB_SELECT::TXHINTS, + DB_SELECT::KNOWNHASHES, DB_SELECT::ZERO_CONF }; @@ -587,14 +587,10 @@ void LMDBBlockDatabase::resetHistoryDatabases() if (Config::DBSettings::getDbType() != ARMORY_DB_TYPE::Super) { auto dbTxouts = getDbPtr(DB_SELECT::TXOUTS); auto dbTxins = getDbPtr(DB_SELECT::TXINS); - auto dbHints = getDbPtr(DB_SELECT::TXHINTS); - auto dbHashes = getDbPtr(DB_SELECT::KNOWNHASHES); closeDatabases(); dbTxouts->eraseOnDisk(); dbTxins->eraseOnDisk(); - dbHints->eraseOnDisk(); - dbHashes->eraseOnDisk(); } openDatabases(); } diff --git a/cppForSwig/gtest/CppBlockUtilsTests.cpp b/cppForSwig/gtest/CppBlockUtilsTests.cpp index 089f10e9b..f2f7f441f 100644 --- a/cppForSwig/gtest/CppBlockUtilsTests.cpp +++ b/cppForSwig/gtest/CppBlockUtilsTests.cpp @@ -2205,6 +2205,112 @@ TEST_F(BlockUtilsFull, TxHints) bdvPtr.reset(); } +//////////////////////////////////////////////////////////////////////////////// +TEST_F(BlockUtilsFull, Load5Blocks_RescanOps) +{ + auto startbdm = [this](BdmInitMode init)->void + { + clients_->init(); + auto bdvID = DBTestUtils::registerBDV( + clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, { + TestChain::scrAddrA, + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrD, + TestChain::scrAddrE, + TestChain::scrAddrF}, + "wallet1", + false); + DBTestUtils::registerWallet(clients_, bdvID, { + TestChain::lb1ScrAddr, + TestChain::lb1ScrAddrP2SH}, + TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet(clients_, bdvID, { + TestChain::lb2ScrAddr, + TestChain::lb2ScrAddrP2SH}, + TestChain::lb2B58ID, + false); + + auto bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(init); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + }; + + auto checkBalance = [](std::shared_ptr bdm) + { + EXPECT_EQ(bdm->blockchain()->top()->getThisHash(), TestChain::blkHash5); + + auto getBal = [bdm](const BinaryData& scrAddr)->uint64_t + { return DBTestUtils::getScrAddrBalance(scrAddr, bdm); }; + + EXPECT_EQ(getBal(TestChain::scrAddrA), 50 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrB), 70 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrC), 20 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrD), 65 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrE), 30 * COIN); + EXPECT_EQ(getBal(TestChain::scrAddrF), 5 * COIN); + + EXPECT_EQ(getBal(TestChain::lb1ScrAddr), 5 * COIN); + EXPECT_EQ(getBal(TestChain::lb1ScrAddrP2SH), 25 * COIN); + EXPECT_EQ(getBal(TestChain::lb2ScrAddr), 30 * COIN); + EXPECT_EQ(getBal(TestChain::lb2ScrAddrP2SH), 0 * COIN); + }; + + auto resetbdm = [this](void)->void + { + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + std::this_thread::sleep_for(1s); + + initBDM(); + }; + + //regular start + startbdm(BdmInitMode::RESUME); + checkBalance(theBDMt_->bdm()); + auto lastScannedRange = theBDMt_->bdm()->getLastScannedRange(); + + //rebuild + resetbdm(); + startbdm(BdmInitMode::REBUILD); + checkBalance(theBDMt_->bdm()); + lastScannedRange = theBDMt_->bdm()->getLastScannedRange(); + + //regular start + resetbdm(); + startbdm(BdmInitMode::RESUME); + checkBalance(theBDMt_->bdm()); + lastScannedRange = theBDMt_->bdm()->getLastScannedRange(); + EXPECT_EQ(lastScannedRange.first, TestChain::blkHash5); + EXPECT_EQ(lastScannedRange.second, TestChain::blkHash5); + + //rescan + resetbdm(); + startbdm(BdmInitMode::RESCAN); + checkBalance(theBDMt_->bdm()); + lastScannedRange = theBDMt_->bdm()->getLastScannedRange(); + EXPECT_EQ(lastScannedRange.first, TestChain::blkHash0); + EXPECT_EQ(lastScannedRange.second, TestChain::blkHash5); + + //regular start + resetbdm(); + startbdm(BdmInitMode::RESUME); + checkBalance(theBDMt_->bdm()); + lastScannedRange = theBDMt_->bdm()->getLastScannedRange(); + EXPECT_EQ(lastScannedRange.first, TestChain::blkHash5); + EXPECT_EQ(lastScannedRange.second, TestChain::blkHash5); +} + /* TODO: - test tx filters From 667c1b65d3cefa23b59fdaaac65846b7ccb0b65e Mon Sep 17 00:00:00 2001 From: goatpig Date: Mon, 7 Sep 2026 11:32:10 +0200 Subject: [PATCH 08/26] cover & fix db rescan for DB_FULL, guard against mangled ZC in mempool --- cppForSwig/TxClasses.cpp | 6 ++++-- cppForSwig/Utils/BtcUtils.cpp | 21 ++++++++++++++------- cppForSwig/ZeroConf/Parser.cpp | 16 ++++++++++++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cppForSwig/TxClasses.cpp b/cppForSwig/TxClasses.cpp index af7b39141..93bfb04e3 100644 --- a/cppForSwig/TxClasses.cpp +++ b/cppForSwig/TxClasses.cpp @@ -591,7 +591,8 @@ Tx Tx::unserialize(const uint8_t* ptr, size_t size) uint32_t nBytes = BtcUtils::TxCalcLength(ptr, size, &txins, &txouts, &witnesses); if (size < 8 || nBytes > size) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "tx size mismatch: expected {} vs actual {}", size, nBytes)); } BinaryDataRef data{ptr, nBytes}; @@ -599,7 +600,8 @@ Tx Tx::unserialize(const uint8_t* ptr, size_t size) bool usesWitness = BtcUtils::checkSwMarker(ptr + 4); uint32_t numWitness = witnesses.size() - 1; if (4 > nBytes - witnesses[numWitness]) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException( + "not enough data to read witness count"); } uint32_t lockTime = READ_UINT32_LE(ptr + witnesses[numWitness]); diff --git a/cppForSwig/Utils/BtcUtils.cpp b/cppForSwig/Utils/BtcUtils.cpp index 98fc73958..5e4f7a45e 100644 --- a/cppForSwig/Utils/BtcUtils.cpp +++ b/cppForSwig/Utils/BtcUtils.cpp @@ -357,7 +357,8 @@ void BtcUtils::TxInCalcLength(const uint8_t* ptr, size_t size, { BinaryRefReader brr(ptr, size); if (brr.getSizeRemaining() < 4) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "txin data is too small: {}", brr.getSizeRemaining())); } // Tx Version @@ -380,7 +381,8 @@ void BtcUtils::TxInCalcLength(const uint8_t* ptr, size_t size, size_t BtcUtils::TxInCalcLength(const uint8_t* ptr, size_t size) { if (size < 37) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "not enough txin data: {}", size)); } uint8_t viLen; size_t scrLen = (size_t)readVarInt(ptr+36, size-36, viLen); @@ -390,7 +392,8 @@ size_t BtcUtils::TxInCalcLength(const uint8_t* ptr, size_t size) size_t BtcUtils::TxOutCalcLength(const uint8_t* ptr, size_t size) { if (size < 9) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "not enough txout data: {}", size)); } uint8_t viLen; @@ -401,7 +404,8 @@ size_t BtcUtils::TxOutCalcLength(const uint8_t* ptr, size_t size) size_t BtcUtils::TxWitnessCalcLength(const uint8_t* ptr, size_t size) { if (size < 1) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "not enough witness data: {}", size)); } size_t witLen = 0; @@ -410,13 +414,15 @@ size_t BtcUtils::TxWitnessCalcLength(const uint8_t* ptr, size_t size) witLen += viStackLen; for (size_t i = 0; i < stackLen; i++) { if (witLen >= size) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "witness data overflow: expected {} vs actual {}", witLen, size)); } uint8_t viLen; witLen += readVarInt(ptr + witLen, size - witLen, viLen); witLen += viLen; if (witLen > size) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "witness data overflow: expected {} vs actual {}", witLen, size)); } } return witLen; @@ -435,7 +441,8 @@ size_t BtcUtils::TxCalcLength(const uint8_t* ptr, size_t size, BinaryRefReader brr(ptr, size); if (brr.getSizeRemaining() < 4) { - throw BtcUtils::BlockDeserializingException(); + throw BtcUtils::BlockDeserializingException(std::format( + "tx data is too small: {}", brr.getSizeRemaining())); } // Tx Version; diff --git a/cppForSwig/ZeroConf/Parser.cpp b/cppForSwig/ZeroConf/Parser.cpp index 44e55359c..32e910d58 100644 --- a/cppForSwig/ZeroConf/Parser.cpp +++ b/cppForSwig/ZeroConf/Parser.cpp @@ -954,11 +954,23 @@ unsigned ZeroConfContainer::loadZeroConfMempool(bool clearMempool) } //Tx, grab it from DB StoredTx zcStx; - db_->getStoredZC(zcStx, zckey); + if (!db_->getStoredZC(zcStx, zckey)) { + LOGDEBUG << std::format("failed to grab zc {:x}", zckey); + continue; + } + if (!zcStx.isInitialized()) { + LOGDEBUG << std::format("zc {:x} is invalid", zckey); + continue; + } //add to newZCMap_ auto parsedTx = std::make_shared(zckey); - parsedTx->setTx(zcStx.getSerializedTx(), zcStx.unixTime); + auto serializedTxData = zcStx.getSerializedTx(); + if (serializedTxData.empty()) { + LOGWARN << std::format("mangled zc in db: {:x}", zckey); + continue; + } + parsedTx->setTx(serializedTxData, zcStx.unixTime); zcMap.emplace(parsedTx->getKey(), std::move(parsedTx)); } else if (keyRef.getSize() == 32) { //tx hash From 47ab5f5b746d227b36bd01b8eaffddaaf24a8888 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sun, 13 Sep 2026 16:40:18 +0200 Subject: [PATCH 09/26] hard check on tx hashes in zc parser pre resolver --- cppForSwig/ZeroConf/Utils.cpp | 11 +- cppForSwig/ZeroConf/Utils.h | 1 - cppForSwig/gtest/ZeroConfTests.cpp | 505 ++++++++++++++++++++++++++++- 3 files changed, 512 insertions(+), 5 deletions(-) diff --git a/cppForSwig/ZeroConf/Utils.cpp b/cppForSwig/ZeroConf/Utils.cpp index 37aa3f680..bc93626ad 100644 --- a/cppForSwig/ZeroConf/Utils.cpp +++ b/cppForSwig/ZeroConf/Utils.cpp @@ -186,8 +186,11 @@ void ZeroConf::preprocessTx(ParsedTx& tx, LMDBBlockDatabase* db, txKey = bd->resolveTxHintCollision(collision); } if (Types::isTxKeyValid(txKey)) { - tx.state = ParsedTxStatus::Mined; - return; + auto txhash = bd->getTxHashForTxKey(txKey); + if (txhash == tx.getTxHash()) { + tx.state = ParsedTxStatus::Mined; + return; + } } const auto& txObj = tx.getTxObj(); @@ -479,6 +482,10 @@ void OutPointRef::resolveDbKey( } if (Types::isTxKeyValid(txKey)) { + auto txHash = bd->getTxHashForTxKey(txKey); + if (txHash != txHash_) { + return; + } setDbKey(txKey); } } diff --git a/cppForSwig/ZeroConf/Utils.h b/cppForSwig/ZeroConf/Utils.h index fa7092c7d..d88d022fc 100644 --- a/cppForSwig/ZeroConf/Utils.h +++ b/cppForSwig/ZeroConf/Utils.h @@ -191,7 +191,6 @@ namespace Armory friend class MempoolSnapshot; public: - //TODO: shouldn't use references for txHashes anymore std::map txHashToDBKey_; // std::map> txMap_; // diff --git a/cppForSwig/gtest/ZeroConfTests.cpp b/cppForSwig/gtest/ZeroConfTests.cpp index f0823d3f7..3f7819ce1 100644 --- a/cppForSwig/gtest/ZeroConfTests.cpp +++ b/cppForSwig/gtest/ZeroConfTests.cpp @@ -410,7 +410,7 @@ class ZeroConfTests_Mempool : public ::testing::Test } ///////////////////////////////////////////////////////////////////////////// - virtual void TearDown(void) + virtual void TearDown() { FileUtils::removeDirectory(blkdir_); FileUtils::removeDirectory(homedir_); @@ -2095,7 +2095,6 @@ TEST_F(ZeroConfTests_FullNode, Replace_ZC_Test) //plot twist: insert fake txhint collision for first spender hash { auto firstTxHash = utxoVec[0].getTxHash(); - std::cout << firstTxHash.toHexStr() << std::endl; ASSERT_EQ(firstTxHash, TestChain::hash32); //insert fake txhint: tx 4.1 @@ -3008,6 +3007,508 @@ TEST_F(ZeroConfTests_FullNode, ZC_InOut_SameBlock) EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 0 * COIN); } +//////////////////////////////////////////////////////////////////////////////// +//add a txhint for a zc that resolves to a valid tx +TEST_F(ZeroConfTests_FullNode, PoisonTxHints) +{ + //setup 5 blocks test chain + TestUtils::setBlocks({ "0", "1", "2", "3", "4", "5" }, blk0dat_); + clients_->init(); + auto bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + std::vector scrAddrVec { + TestChain::scrAddrA, + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrE + }; + + const std::vector lb1ScrAddrs { + TestChain::lb1ScrAddr, + TestChain::lb1ScrAddrP2SH + }; + const std::vector lb2ScrAddrs { + TestChain::lb2ScrAddr, + TestChain::lb2ScrAddrP2SH + }; + + Wallets::IO::CreateWalletParams params{ + homedir_, + Passphrase::SetNew{1ms, 0, {}}, + Passphrase::SetNew{1ms, 0, {}}, + nullptr, 10 + }; + std::unique_ptr seed( + new Seeds::ClearTextSeed_Armory()); + auto assetWlt = Wallets::AssetWallet_Single::createFromSeed( + std::move(seed), params); + + //register with db + std::vector addrVec; + auto hashSet = assetWlt->getAddrHashSet(); + std::vector hashVec; + hashVec.insert(hashVec.begin(), hashSet.begin(), hashSet.end()); + + DBTestUtils::registerWallet(clients_, bdvID, hashVec, assetWlt->getID(), + false); + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + auto bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + auto bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(bdm.get(), DB_SELECT::HEADERS), 5U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash5); + EXPECT_TRUE(bdm->blockchain()->getHeaderByHash(TestChain::blkHash5)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 25 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 0 * COIN); + + //add a zc + BinaryData ZCHash; + { + ////spend 27 from wlt to assetWlt's first 2 unused addresses + ////send rest back to scrAddrA + + auto spendVal = 27 * COIN; + Signing::Signer signer; + + //instantiate resolver feed overloaded object + auto feed = std::make_shared(); + feed->addPrivKey(TestChain::privKeyAddrB.getRef()); + feed->addPrivKey(TestChain::privKeyAddrC.getRef()); + feed->addPrivKey(TestChain::privKeyAddrD.getRef()); + feed->addPrivKey(TestChain::privKeyAddrE.getRef()); + + //get utxo list for spend value + auto unspentVec = DBTestUtils::getUTXOsForScrAddrs(bdm, { + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrE + }); + + std::vector utxoVec; + uint64_t tval = 0; + auto utxoIter = unspentVec.begin(); + while (utxoIter != unspentVec.end()) { + tval += utxoIter->getAmount(); + utxoVec.push_back(*utxoIter); + + if (tval > spendVal) { + break; + } + ++utxoIter; + } + + //create script spender objects + uint64_t total = 0; + for (auto& utxo : utxoVec) { + total += utxo.getAmount(); + signer.addSpender(getSpenderPtr(utxo, true)); + } + + //spend 12 to first address + auto addr0 = assetWlt->getNewAddress(); + signer.addRecipient(addr0->getRecipient(12 * COIN)); + addrVec.push_back(addr0->getPrefixedHash()); + + //spend 15 to addr 1, use P2PKH + auto addr1 = assetWlt->getNewAddress(); + signer.addRecipient(addr1->getRecipient(15 * COIN)); + addrVec.push_back(addr1->getPrefixedHash()); + + if (total > spendVal) { + //deal with change, no fee + auto changeVal = total - spendVal; + auto recipientChange = std::make_shared( + TestChain::scrAddrD.getSliceCopy(1, 20), changeVal); + signer.addRecipient(recipientChange); + } + + //sign, verify then broadcast + signer.setFeed(feed); + signer.sign(); + EXPECT_TRUE(signer.verify()); + + auto rawTx = signer.serializeSignedTx(); + DBTestUtils::ZcVector zcVec; + zcVec.push_back(rawTx, 14000000); + + //poison txhints with the zchash + ZCHash = std::move(BtcUtils::getHash256(rawTx)); + { + const uint64_t blockIDMask = 0x00000000FFFFFFFF; + uint8_t hashTableIndex = ZCHash.getPtr()[8]; + auto db = bdm->getIFace(); + auto tx = db->beginHashTableTx( + DB_SELECT::TXHINTS, hashTableIndex, LMDB::Mode::ReadWrite); + + uint64_t blockID = 5ul << 32; + uint64_t txHintKey; + std::memcpy(&txHintKey, ZCHash.getPtr(), 8); + txHintKey = (txHintKey & blockIDMask) | blockID; + std::vector txids{1}; + tx->insert( + LMDB::DataRef{ + sizeof(uint64_t), + (const char*)&txHintKey}, + LMDB::DataRef{ + sizeof(uint16_t) * txids.size(), + (const char*)&txids[0]} + ); + } + + DBTestUtils::pushNewZc(theBDMt_, zcVec); + auto txioVec = DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + EXPECT_EQ(txioVec.first.size(), 4ULL); + EXPECT_EQ(txioVec.second.size(), 0ULL); + + for (const auto& txio : txioVec.first) { + if (txio.hasTxOutZC()) { + auto txObj = DBTestUtils::getTxByKey(clients_, bdvID, + txio.getTxKeyOfOutput()); + EXPECT_EQ(txObj.getThisHash(), ZCHash); + } + if (txio.hasTxInZC()) { + auto txObj = DBTestUtils::getTxByKey(clients_, bdvID, + txio.getTxKeyOfInput()); + EXPECT_EQ(txObj.getThisHash(), ZCHash); + } + } + } + + //check balances + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrE, bdm), 0 * COIN); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[0], bdm), 12 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[1], bdm), 15 * COIN); + + //cleanup + bdvPtr.reset(); +} + +//////////////////////////////////////////////////////////////////////////////// +//add a zc, add a txhint for that hash that points to a valid mined tx, +//then try to spend from first zc +TEST_F(ZeroConfTests_FullNode, PoisonTxHints2) +{ + //setup 5 blocks test chain + TestUtils::setBlocks({ "0", "1", "2", "3", "4", "5" }, blk0dat_); + clients_->init(); + auto bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + std::vector scrAddrVec { + TestChain::scrAddrA, + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrE + }; + + const std::vector lb1ScrAddrs { + TestChain::lb1ScrAddr, + TestChain::lb1ScrAddrP2SH + }; + const std::vector lb2ScrAddrs { + TestChain::lb2ScrAddr, + TestChain::lb2ScrAddrP2SH + }; + + Wallets::IO::CreateWalletParams params{ + homedir_, + Passphrase::SetNew{1ms, 0, {}}, + Passphrase::SetNew{1ms, 0, {}}, + nullptr, 10 + }; + std::unique_ptr seed( + new Seeds::ClearTextSeed_Armory()); + auto assetWlt = Wallets::AssetWallet_Single::createFromSeed( + std::move(seed), params); + + //register with db + std::vector addrVec; + auto hashSet = assetWlt->getAddrHashSet(); + std::vector hashVec; + hashVec.insert(hashVec.begin(), hashSet.begin(), hashSet.end()); + + DBTestUtils::registerWallet(clients_, bdvID, hashVec, assetWlt->getID(), + false); + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + auto bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + auto bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(bdm.get(), DB_SELECT::HEADERS), 5U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash5); + EXPECT_TRUE(bdm->blockchain()->getHeaderByHash(TestChain::blkHash5)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 25 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 0 * COIN); + + //add a zc + BinaryData ZCHash1; + { + ////spend 27 from wlt to assetWlt's first 2 unused addresses + ////send rest back to scrAddrA + + auto spendVal = 27 * COIN; + Signing::Signer signer; + + //instantiate resolver feed overloaded object + auto feed = std::make_shared(); + feed->addPrivKey(TestChain::privKeyAddrB.getRef()); + feed->addPrivKey(TestChain::privKeyAddrC.getRef()); + feed->addPrivKey(TestChain::privKeyAddrD.getRef()); + feed->addPrivKey(TestChain::privKeyAddrE.getRef()); + + //get utxo list for spend value + auto unspentVec = DBTestUtils::getUTXOsForScrAddrs(bdm, { + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrE + }); + + std::vector utxoVec; + uint64_t tval = 0; + auto utxoIter = unspentVec.begin(); + while (utxoIter != unspentVec.end()) { + tval += utxoIter->getAmount(); + utxoVec.push_back(*utxoIter); + + if (tval > spendVal) { + break; + } + ++utxoIter; + } + + //create script spender objects + uint64_t total = 0; + for (auto& utxo : utxoVec) { + total += utxo.getAmount(); + signer.addSpender(getSpenderPtr(utxo, true)); + } + + //spend 12 to first address + auto addr0 = assetWlt->getNewAddress(); + signer.addRecipient(addr0->getRecipient(12 * COIN)); + addrVec.push_back(addr0->getPrefixedHash()); + + //spend 15 to addr 1, use P2PKH + auto addr1 = assetWlt->getNewAddress(); + signer.addRecipient(addr1->getRecipient(15 * COIN)); + addrVec.push_back(addr1->getPrefixedHash()); + + if (total > spendVal) { + //deal with change, no fee + auto changeVal = total - spendVal; + auto recipientChange = std::make_shared( + TestChain::scrAddrD.getSliceCopy(1, 20), changeVal); + signer.addRecipient(recipientChange); + } + + //sign, verify then broadcast + signer.setFeed(feed); + signer.sign(); + EXPECT_TRUE(signer.verify()); + + auto rawTx = signer.serializeSignedTx(); + DBTestUtils::ZcVector zcVec; + zcVec.push_back(rawTx, 14000000); + + ZCHash1 = std::move(BtcUtils::getHash256(rawTx)); + DBTestUtils::pushNewZc(theBDMt_, zcVec); + auto txioVec = DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + EXPECT_EQ(txioVec.first.size(), 4ULL); + EXPECT_EQ(txioVec.second.size(), 0ULL); + + for (const auto& txio : txioVec.first) { + if (txio.hasTxOutZC()) { + auto txObj = DBTestUtils::getTxByKey(clients_, bdvID, + txio.getTxKeyOfOutput()); + EXPECT_EQ(txObj.getThisHash(), ZCHash1); + } + if (txio.hasTxInZC()) { + auto txObj = DBTestUtils::getTxByKey(clients_, bdvID, + txio.getTxKeyOfInput()); + EXPECT_EQ(txObj.getThisHash(), ZCHash1); + } + } + } + + //check balances + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrE, bdm), 0 * COIN); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[0], bdm), 12 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[1], bdm), 15 * COIN); + + //poison txhints with the zchash + { + const uint64_t blockIDMask = 0x00000000FFFFFFFF; + uint8_t hashTableIndex = ZCHash1.getPtr()[8]; + auto db = bdm->getIFace(); + auto tx = db->beginHashTableTx( + DB_SELECT::TXHINTS, hashTableIndex, LMDB::Mode::ReadWrite); + + uint64_t blockID = 5ul << 32; + uint64_t txHintKey; + std::memcpy(&txHintKey, ZCHash1.getPtr(), 8); + txHintKey = (txHintKey & blockIDMask) | blockID; + std::vector txids{1}; + tx->insert( + LMDB::DataRef{ + sizeof(uint64_t), + (const char*)&txHintKey}, + LMDB::DataRef{ + sizeof(uint16_t) * txids.size(), + (const char*)&txids[0]} + ); + } + + //spend from the zc + BinaryData ZCHash2; + { + ////spend 27 from wlt to assetWlt's first 2 unused addresses + ////send rest back to scrAddrA + + auto spendVal = 18 * COIN; + Signing::Signer signer; + + //instantiate resolver + auto assetFeed = std::make_shared(assetWlt); + + //get utxo list for spend value + auto unspentVec = DBTestUtils::getZCUTXOs(bdm, { + addrVec[0], + addrVec[1]} + ); + + std::vector utxoVec; + uint64_t tval = 0; + auto utxoIter = unspentVec.begin(); + while (utxoIter != unspentVec.end()) { + tval += utxoIter->getAmount(); + utxoVec.push_back(*utxoIter); + + if (tval > spendVal) { + break; + } + ++utxoIter; + } + + //create script spender objects + uint64_t total = 0; + for (auto& utxo : utxoVec) { + total += utxo.getAmount(); + signer.addSpender(getSpenderPtr(utxo, true)); + } + + //spend 10 to third address + auto addr2 = assetWlt->getNewAddress(); + signer.addRecipient(addr2->getRecipient(10 * COIN)); + addrVec.push_back(addr2->getPrefixedHash()); + + //spend 8 to addr3 + auto addr3 = assetWlt->getNewAddress(); + signer.addRecipient(addr3->getRecipient(8 * COIN)); + addrVec.push_back(addr3->getPrefixedHash()); + + if (total > spendVal) { + //deal with change, no fee + auto changeVal = total - spendVal; + auto recipientChange = std::make_shared( + TestChain::scrAddrD.getSliceCopy(1, 20), changeVal); + signer.addRecipient(recipientChange); + } + + //sign, verify then broadcast + { + auto lock = assetWlt->lockDecryptedContainer({}); + signer.setFeed(assetFeed); + signer.sign(); + EXPECT_TRUE(signer.verify()); + } + + auto rawTx = signer.serializeSignedTx(); + DBTestUtils::ZcVector zcVec; + zcVec.push_back(rawTx, 15000000); + + ZCHash2 = std::move(BtcUtils::getHash256(rawTx)); + DBTestUtils::pushNewZc(theBDMt_, zcVec); + auto txioVec = DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + EXPECT_EQ(txioVec.first.size(), 4ULL); + EXPECT_EQ(txioVec.second.size(), 0ULL); + + for (const auto& txio : txioVec.first) { + if (txio.hasTxOutZC()) { + auto txObj = DBTestUtils::getTxByKey(clients_, bdvID, + txio.getTxKeyOfOutput()); + if (txio.hasTxInZC()) { + EXPECT_EQ(txObj.getThisHash(), ZCHash1); + } else { + EXPECT_EQ(txObj.getThisHash(), ZCHash2); + } + } + if (txio.hasTxInZC()) { + auto txObj = DBTestUtils::getTxByKey(clients_, bdvID, + txio.getTxKeyOfInput()); + EXPECT_EQ(txObj.getThisHash(), ZCHash2); + } + } + } + + //check balances + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrE, bdm), 0 * COIN); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[0], bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[1], bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[2], bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(addrVec[3], bdm), 8 * COIN); + + //cleanup + bdvPtr.reset(); +} + //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //NOTE: have to rework supernode scanner before getting to this From 526e3348a3753a642b611d1b68eba70f7faee3e9 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sat, 19 Sep 2026 13:35:20 +0200 Subject: [PATCH 10/26] improve build instructions windows: fix cmake defintion, fix name clash in capnp schema fix wallet detection in TxFramesOffline.py --- armoryengine/WalletUtils.py | 6 ++++ build_instructions/linux/linux.md | 6 ++-- .../windows/Windows_build_notes.md | 28 +++++++++---------- cppForSwig/BDM_Server.cpp | 4 +-- cppForSwig/CMakeLists.txt | 2 +- cppForSwig/DBClientClasses.cpp | 12 ++------ cppForSwig/capnp/BDV.capnp | 19 ++++++------- cppForSwig/gtest/TestUtils.cpp | 3 +- ui/TxFramesOffline.py | 23 +++++---------- 9 files changed, 48 insertions(+), 55 deletions(-) diff --git a/armoryengine/WalletUtils.py b/armoryengine/WalletUtils.py index b555601cb..83d71a270 100644 --- a/armoryengine/WalletUtils.py +++ b/armoryengine/WalletUtils.py @@ -178,6 +178,12 @@ def getWltForScrAddr(self, scrAddr): return iterWlt return None + def getWltForAddrStr(self, addrStr): + for _, iterWlt in self._walletMap.items(): + if iterWlt.hasAddrString(addrStr): + return iterWlt + return None + def hasWallet(self, wltId: str): return wltId in self._wltIdToDbId diff --git a/build_instructions/linux/linux.md b/build_instructions/linux/linux.md index 2bd6a8c46..903a83611 100644 --- a/build_instructions/linux/linux.md +++ b/build_instructions/linux/linux.md @@ -39,8 +39,10 @@ The following dependencies you have to build from source. ``` git clone https://github.com/warmcat/libwebsockets cd libwebsockets - mkdir build & cd build - cmake -DLWS_WITH_SSL=OFF .. + git checkout v4.5.8 + mkdir build + cmake -DLWS_WITH_SSL=OFF -DLWS_WITHOUT_TESTAPPS=ON -B build + cd build make ``` diff --git a/build_instructions/windows/Windows_build_notes.md b/build_instructions/windows/Windows_build_notes.md index 3146833af..69273aa98 100644 --- a/build_instructions/windows/Windows_build_notes.md +++ b/build_instructions/windows/Windows_build_notes.md @@ -1,19 +1,14 @@ # 1. Installing pre-requisites * MSVC (Visual Studio Community): https://visualstudio.microsoft.com/downloads/ * MSYS2: ONLY DOWNLOAD the installer of MSYS2 from https://www.msys2.org/ and FOLLOW the setup instructions in the current document -* Python <=3.12 (3.13 breaks `pycapnp`) -* Git for Windows: https://gitforwindows.org/ +* Python (tested on 3.14.7) # 2. Installing build tools As you will be exclusively using **MSYS2 MINGW64**, make sure you have opened that and not the UCRT/MSYS/CLANG terminal. ``` pacman -Syu -pacman -S autoconf automake libtoolize mingw-w64-x86_64-gcc mingw-w64-x86_64-libevent mingw-w64-x86_64-make mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja git +pacman -S autoconf automake libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-libevent mingw-w64-x86_64-make mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja git ``` -Next, create a symlink for the cmake binary in order to use the cmake command in MinGW64: - -1. Open Windows Command Prompt and navigate to `\mingw64\bin` -2. Run `mklink make mingw32-make.exe` # 3. Installing Python dependencies ``` @@ -45,14 +40,17 @@ It is strongly recommended to pick a single folder in which you will download al git clone https://github.com/warmcat/libwebsockets.git cd libwebsockets git checkout v4.3.3 - mkdir build & cd build - cmake -G Ninja -DLWS_WITH_SSL=OFF .. + mkdir build + cmake -G Ninja -DLWS_WITH_SSL=OFF -DLWS_WITHOUT_TESTAPPS=ON -DDISABLE_WERROR=ON -B build + cd build ninja ``` 3. [LMDB](https://github.com/LMDB/lmdb): **Note**: Make sure you build off of the mdb.master branch, or else mmap will eat up all your free disk space! ``` git clone https://github.com/LMDB/lmdb.git + cb lmdb + git checkout mdb.master cd libraries/liblmdb make ``` @@ -61,17 +59,19 @@ It is strongly recommended to pick a single folder in which you will download al ``` git clone https://github.com/capnproto/capnproto.git cd capnproto - git checkout v1.0.2 - mkdir build & cd build - cmake -G Ninja .. + git checkout v1.5.0 + mkdir build + cmake -G Ninja -B build + cd build ninja ``` # 5. Building BitcoinArmory ``` git clone https://github.com/goatpig/BitcoinArmory cd BitcoinArmory -mkdir build & cd build -cmake -G Ninja .. +mkdir build +cmake -G Ninja -B build +cd build ninja ``` > [!WARNING] diff --git a/cppForSwig/BDM_Server.cpp b/cppForSwig/BDM_Server.cpp index 797e9c747..25a238968 100644 --- a/cppForSwig/BDM_Server.cpp +++ b/cppForSwig/BDM_Server.cpp @@ -741,11 +741,11 @@ void BDV_Server_Object::init() auto notifList = notifs.initNotifs(1); auto notif = notifList[0]; if (ready) { - auto readyNotif = notif.initReady(); + auto readyNotif = notif.initConnectionIsReady(); readyNotif.setHeight(blockchain().top()->getBlockHeight()); readyNotif.setBranchHeight(UINT32_MAX); } else { - notif.setRegistered(); + notif.setRegistrationDone(); } //we expect this message to be smaller than our scratchpad diff --git a/cppForSwig/CMakeLists.txt b/cppForSwig/CMakeLists.txt index d160fcbe2..6d5fe536a 100644 --- a/cppForSwig/CMakeLists.txt +++ b/cppForSwig/CMakeLists.txt @@ -225,7 +225,7 @@ ADD_LIBRARY(BridgeLib BridgeAPI/Wallets/Manager.cpp BridgeAPI/Wallets/Notifications.cpp BridgeAPI/Wallets/TxIOCache.cpp - BridgeAPI/BlockchainDbClient.cpp + BridgeAPI/DBSetup.cpp ) TARGET_INCLUDE_DIRECTORIES(BridgeLib diff --git a/cppForSwig/DBClientClasses.cpp b/cppForSwig/DBClientClasses.cpp index 0d0296d64..1e3fa2e03 100644 --- a/cppForSwig/DBClientClasses.cpp +++ b/cppForSwig/DBClientClasses.cpp @@ -204,17 +204,17 @@ bool RemoteCallback::processNotifications( break; } - case BDV::Notification::READY: + case BDV::Notification::CONNECTION_IS_READY: { BdmNotification bdmNotif(BDMAction_Ready); - auto newBlock = notif.getReady(); + auto newBlock = notif.getConnectionIsReady(); bdmNotif.newBlock = NewBlockNotif{ newBlock.getHeight(), newBlock.getBranchHeight(), {}, {}}; run(std::move(bdmNotif)); break; } - case BDV::Notification::REGISTERED: + case BDV::Notification::REGISTRATION_DONE: { BdmNotification bdmNotif(BDMAction_Registered); run(std::move(bdmNotif)); @@ -237,12 +237,6 @@ bool RemoteCallback::processNotifications( break; } - case BDV::Notification::TERMINATE: - { - //shut down command from server - return false; - } - case BDV::Notification::NODE_STATUS: { BdmNotification bdmNotif(BDMAction_NodeStatus); diff --git a/cppForSwig/capnp/BDV.capnp b/cppForSwig/capnp/BDV.capnp index b0d971ff3..3d563e124 100644 --- a/cppForSwig/capnp/BDV.capnp +++ b/cppForSwig/capnp/BDV.capnp @@ -166,16 +166,15 @@ struct Notification { requestId @0 : Text; union { - terminate @1 : Void; - registered @2 : Void; - ready @3 : Types.NewBlockNotif; - newBlock @4 : Types.NewBlockNotif; - zc @5 : List(Types.TxioPair); - invalidatedZc @6 : List(Data); - refresh @7 : Refresh; - nodeStatus @8 : Types.NodeStatus; - progress @9 : Types.ScanProgress; - error @10: ServerError; + registrationDone @1 : Void; + connectionIsReady @2 : Types.NewBlockNotif; + newBlock @3 : Types.NewBlockNotif; + zc @4 : List(Types.TxioPair); + invalidatedZc @5 : List(Data); + refresh @6 : Refresh; + nodeStatus @7 : Types.NodeStatus; + progress @8 : Types.ScanProgress; + error @9 : ServerError; } } diff --git a/cppForSwig/gtest/TestUtils.cpp b/cppForSwig/gtest/TestUtils.cpp index 190bcf1a0..173b03a33 100644 --- a/cppForSwig/gtest/TestUtils.cpp +++ b/cppForSwig/gtest/TestUtils.cpp @@ -550,7 +550,8 @@ namespace DBTestUtils ///////////////////////////////////////////////////////////////////////////// void waitOnBDVReady(Clients* clients, Types::BdvId bdvId) { - waitOnSignal(clients, bdvId, (int)Codec::BDV::Notification::READY); + waitOnSignal(clients, bdvId, + (int)Codec::BDV::Notification::CONNECTION_IS_READY); } ///////////////////////////////////////////////////////////////////////////// diff --git a/ui/TxFramesOffline.py b/ui/TxFramesOffline.py index ca5ee1c57..708e3c525 100644 --- a/ui/TxFramesOffline.py +++ b/ui/TxFramesOffline.py @@ -277,9 +277,9 @@ def processUSTX(self): # Collect the input wallets (hopefully just one of them) fromWlts = set() for addrStr, amt, a, b, c, script in data[FIELDS.InList]: - wltID = self.main.wallets.getWltForScrAddr(addrStr) - if wltID: - fromWlts.add(wltID) + wlt = self.main.wallets.getWltForAddrStr(addrStr) + if wlt: + fromWlts.add(wlt.dbId) if len(fromWlts) > 1: QtWidgets.QMessageBox.warning(self, self.tr('Multiple Input Wallets'), \ @@ -298,22 +298,18 @@ def processUSTX(self): return spendWltID = fromWlts.pop() - self.wlt = self.main.walletMap[spendWltID] + self.wlt = self.main.wallets.get(spendWltID) - toWlts = set() myOutSum = 0 theirOutSum = 0 rvPairs = [] idx = 0 for scrType, amt, binScript, multiSigList in data[FIELDS.OutList]: recip = script_to_scrAddr(binScript) - try: - wltID = self.main.getWalletForAddrHash(recip) - except BadAddressError: - wltID = '' + recipWlt = self.main.wallets.getWltForScrAddr(recip) + wltID = recipWlt.dbId if recipWlt else None if wltID == spendWltID: - toWlts.add(wltID) myOutSum += amt self.idxSelf.append(idx) else: @@ -332,10 +328,8 @@ def processUSTX(self): self.leValue = theirOutSum self.makeReviewFrame() - ############################################################################ def makeReviewFrame(self): - # ## if self.ustxObj == None: self.infoLbls[0][2].setText('') self.infoLbls[1][2].setText('') @@ -347,7 +341,7 @@ def makeReviewFrame(self): ##### 1 if self.wlt: - self.infoLbls[0][2].setText(self.wlt.uniqueIDB58) + self.infoLbls[0][2].setText(self.wlt.walletId) self.infoLbls[1][2].setText(self.wlt.labelName) else: self.infoLbls[0][2].setText(self.tr('[[ Unrelated ]]')) @@ -365,7 +359,6 @@ def makeReviewFrame(self): self.moreInfo.setVisible(True) def execMoreTxInfo(self): - if not self.ustxObj: self.processUSTX() @@ -379,8 +372,6 @@ def execMoreTxInfo(self): precomputeIdxGray=self.idxSelf, precomputeAmt=leVal, txtime=-1) dlgTxInfo.exec_() - - def signTx(self): if not self.ustxObj: QtWidgets.QMessageBox.critical(self, self.tr('Cannot Sign'), \ From 1fdc96949c546f6219d57d767bc22ecbe9cb5b21 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sun, 20 Sep 2026 15:34:06 +0200 Subject: [PATCH 11/26] make windows build easier --- armoryengine/CppBridge.py | 2 +- cppForSwig/AsyncClient.cpp | 2 +- cppForSwig/AsyncClient.h | 2 +- cppForSwig/BridgeAPI/BridgeMain.cpp | 12 +- cppForSwig/BridgeAPI/BridgeSocket.cpp | 4 +- cppForSwig/BridgeAPI/BridgeSocket.h | 2 +- cppForSwig/BridgeAPI/CppBridge.cpp | 5 +- cppForSwig/BridgeAPI/CppBridge.h | 8 +- cppForSwig/BridgeAPI/DBSetup.cpp | 67 +++-- cppForSwig/BridgeAPI/DBSetup.h | 2 +- cppForSwig/BridgeAPI/ProtoCommandParser.cpp | 22 +- cppForSwig/Network/SocketObject.cpp | 35 ++- cppForSwig/Network/SocketObject.h | 24 +- cppForSwig/Network/StringSockets.cpp | 2 +- cppForSwig/Network/StringSockets.h | 2 +- cppForSwig/Network/WebSocketClient.cpp | 16 +- cppForSwig/Network/WebSocketClient.h | 2 +- cppForSwig/Node/BitcoinP2P.cpp | 4 +- cppForSwig/Node/BitcoinP2P.h | 6 +- cppForSwig/Server.cpp | 6 +- cppForSwig/Utils/ArmoryConfig.cpp | 265 +++++++------------- cppForSwig/Utils/ArmoryConfig.h | 23 +- cppForSwig/Utils/BtcUtils.cpp | 2 +- cppForSwig/capnp/Bridge.capnp | 2 +- cppForSwig/gtest/BridgeTests.cpp | 72 +++--- qtdialogs/setupmanager/DatabaseTab.py | 8 +- qtdialogs/setupmanager/DlgSetupManager.py | 20 +- 27 files changed, 270 insertions(+), 347 deletions(-) diff --git a/armoryengine/CppBridge.py b/armoryengine/CppBridge.py index 620fd3dd3..c188acd33 100755 --- a/armoryengine/CppBridge.py +++ b/armoryengine/CppBridge.py @@ -426,7 +426,7 @@ def __init__(self, bridgeSocket): ############################################################################# ## commands ## def connectToIp(self, - ip: str, port: str, callbackId: str, + ip: str, port: int, callbackId: str, resultCallback: callable = None): """ Connect to remote DB by IP address (1-way auth). diff --git a/cppForSwig/AsyncClient.cpp b/cppForSwig/AsyncClient.cpp index bbdcce298..278d9f811 100755 --- a/cppForSwig/AsyncClient.cpp +++ b/cppForSwig/AsyncClient.cpp @@ -227,7 +227,7 @@ void BlockDataViewer::addPublicKey(const SecureBinaryData& pubkey, bool oneWay) /////////////////////////////////////////////////////////////////////////////// std::shared_ptr BlockDataViewer::getNewBDV( - const std::string& addr, const std::string& port, + const std::string& addr, port_t port, std::shared_ptr peers, bool oneWayAuth, std::shared_ptr callbackPtr) { diff --git a/cppForSwig/AsyncClient.h b/cppForSwig/AsyncClient.h index 6614888aa..7a9f90818 100755 --- a/cppForSwig/AsyncClient.h +++ b/cppForSwig/AsyncClient.h @@ -167,7 +167,7 @@ namespace AsyncClient //setup static std::shared_ptr getNewBDV( - const std::string&, const std::string&, + const std::string&, Armory::Network::port_t, std::shared_ptr, bool, std::shared_ptr); diff --git a/cppForSwig/BridgeAPI/BridgeMain.cpp b/cppForSwig/BridgeAPI/BridgeMain.cpp index 8313c4187..63c215260 100644 --- a/cppForSwig/BridgeAPI/BridgeMain.cpp +++ b/cppForSwig/BridgeAPI/BridgeMain.cpp @@ -55,7 +55,13 @@ int main(int argc, char* argv[]) LOGERR << "could not find bridge port env var, aborting!"; exit(-2); } - std::string bridgePortStr(bridgePortChar); + Armory::Network::port_t bridgePort; + auto [ptr, ec] = std::from_chars(bridgePortChar, + bridgePortChar + strlen(bridgePortChar), bridgePort); + if (ec != std::errc{}) { + LOGERR << "invalid bridge port, aborting"; + exit(-3); + } //init static configuration variables Armory::Config::parseArgs(count, args, @@ -72,12 +78,12 @@ int main(int argc, char* argv[]) "\n - offline: " << Armory::Config::NetworkSettings::isOffline() << "\n - auth mode: " << Armory::Config::NetworkSettings::oneWayAuth() << "\n - db port: " << Armory::Config::NetworkSettings::dbPort() << - "\n - bridge port: " << bridgePortStr; + "\n - bridge port: " << bridgePort; //setup the bridge & socket auto bridge = std::make_shared(); auto sockPtr = std::make_shared( - "127.0.0.1", bridgePortStr, bridge); + "127.0.0.1", bridgePort, bridge); //set bridge write lambda auto pushPayloadLbd = [sockPtr]( diff --git a/cppForSwig/BridgeAPI/BridgeSocket.cpp b/cppForSwig/BridgeAPI/BridgeSocket.cpp index 15a07b4c7..49af4488b 100755 --- a/cppForSwig/BridgeAPI/BridgeSocket.cpp +++ b/cppForSwig/BridgeAPI/BridgeSocket.cpp @@ -28,10 +28,10 @@ using namespace Armory::Bridge; //// //////////////////////////////////////////////////////////////////////////////// CppBridgeSocket::CppBridgeSocket( - const std::string& addr, const std::string& port, + const std::string& addr, Network::port_t port, std::shared_ptr bridgePtr) : PersistentSocket(addr, port), bridgePtr_(bridgePtr), - serverName_(addr + ":" + port) + serverName_(std::format("{}:{}", addr, port)) { //setup auth peers db peers_ = std::make_shared(); diff --git a/cppForSwig/BridgeAPI/BridgeSocket.h b/cppForSwig/BridgeAPI/BridgeSocket.h index 5649fc5d3..dec3ff719 100755 --- a/cppForSwig/BridgeAPI/BridgeSocket.h +++ b/cppForSwig/BridgeAPI/BridgeSocket.h @@ -58,7 +58,7 @@ namespace Armory public: CppBridgeSocket( - const std::string&, const std::string&, + const std::string&, Network::port_t, std::shared_ptr); SocketType type(void) const override; diff --git a/cppForSwig/BridgeAPI/CppBridge.cpp b/cppForSwig/BridgeAPI/CppBridge.cpp index de36258fd..d0eaae427 100755 --- a/cppForSwig/BridgeAPI/CppBridge.cpp +++ b/cppForSwig/BridgeAPI/CppBridge.cpp @@ -1391,7 +1391,7 @@ void CppBridge::setPeerLabel( //////////////////////////////////////////////////////////////////////////////// // db connection routines -void CppBridge::connectToIp(const std::string& ip, const std::string& port, +void CppBridge::connectToIp(const std::string& ip, Network::port_t port, const CallbackId& callbackId, MessageId refId) { /* @@ -1616,9 +1616,8 @@ void CppBridge::runAutomationContext(CallbackId cbId, MessageId refId) //connect to db notifyFunc(AutomationStep::ConnectToDb); auto peers = automationContext_->getPeerStore(); - auto port = std::to_string(automationContext_->getDbPort()); bdvPtr_ = setupClientConnection(peers, - "127.0.0.1", port, + "127.0.0.1", automationContext_->getDbPort(), false, nullptr, wltManager_->getBdvCallback()); if (bdvPtr_ == nullptr) { diff --git a/cppForSwig/BridgeAPI/CppBridge.h b/cppForSwig/BridgeAPI/CppBridge.h index 4caaa21b5..a022ae8d8 100755 --- a/cppForSwig/BridgeAPI/CppBridge.h +++ b/cppForSwig/BridgeAPI/CppBridge.h @@ -156,7 +156,7 @@ namespace Armory void cleanupAutomationContext(CallbackId, MessageId); //db setup - void connectToIp(const std::string&, const std::string&, + void connectToIp(const std::string&, uint16_t, const CallbackId&, MessageId); void connectToPeer(const std::string&, MessageId); void beginDbSession(void); @@ -173,7 +173,7 @@ namespace Armory //wallet registration void registerWallets(void); void registerWallet(const Wallets::WalletId&, - const Wallets::AddressAccountId&, bool isNew); + const Wallets::AddressAccountId&, bool); BinaryData getNodeStatus(MessageId); //balance and counts @@ -207,8 +207,8 @@ namespace Armory void forkWatchingOnly(const Wallets::WalletId&, const CallbackId&, MessageId); void exportKeys(const Wallets::WalletId&, - const Wallets::AddressAccountId&, bool includePrivateKeys, - const CallbackId& callbackId, MessageId); + const Wallets::AddressAccountId&, bool, + const CallbackId&, MessageId); //ledgers const std::string& getLedgerDelegateId(void); diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index 0b3d190da..47a48c56b 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -61,7 +61,7 @@ namespace { commandLine.append(std::format("{} ", arg)); } - //mandatory, process handle is writting in pi after start + //mandatory, process handle is written inside pi after start STARTUPINFOW si; ZeroMemory( &si, sizeof(si) ); si.cb = sizeof(si); @@ -94,12 +94,11 @@ namespace { &si, &pi )) { auto lastError = GetLastError(); - throw std::runtime_error("failed to spawn ArmorDB with error: " + std::to_string(lastError)); + return { {}, lastError }; } - } + } auto handle = pi.hProcess; CloseHandle(pi.hThread); - return { {handle}, {} }; } @@ -117,6 +116,7 @@ namespace { std::pair spawnProcess( const std::filesystem::path& target, + const std::map& pathArgs, const std::vector& args, const std::map& envvars, bool captureStdOut) @@ -124,9 +124,21 @@ namespace { std::vector argv; auto targetStr = target.string(); argv.emplace_back(targetStr.data()); + + //path args + std::vector pathArgsStr; + for (const auto& pathArg : pathArgs) { + pathArgsStr.emplace_back(std::format("{}={}", + pathArg.first, pathArg.second.string())); + argv.emplace_back((char*)pathArgsStr.back().data()); + } + + //other args for (const auto& arg : args) { argv.emplace_back((char*)arg.data()); } + + //argv null terminator argv.emplace_back(nullptr); std::vector envStrings; @@ -192,19 +204,21 @@ namespace { #endif - std::pair getIpAndPortFromPeerName( + std::pair getIpAndPortFromPeerName( const std::string& peerName) { //TODO: flesh this out std::stringstream ss(peerName); - std::pair output; + std::pair output; //ip std::getline(ss, output.first, ':'); //port if (ss.good()) { - std::getline(ss, output.second); + std::string portStr; + std::getline(ss, portStr); + output.second = std::stoi(portStr); } else { output.second = Config::NetworkSettings::dbPort(); } @@ -349,14 +363,13 @@ namespace { //is this a fully qualified path? targetDir = std::filesystem::absolute(dataDir); } - auto dataDirStr = std::format("--datadir={}", targetDir.string()); //run bitcoind --version - std::vector args{ - std::format("--datadir={}", targetDir.string()), - {"--version"} - }; - auto result = spawnProcess(binPath, args, {}, true); + auto result = spawnProcess(binPath, + {{"--datadir", targetDir}}, + {"--version"}, + {}, true + ); result.first.wait(); if (dataDir.empty()) { @@ -409,7 +422,7 @@ namespace { //////////////////////////////////////////////////////////////////////////////// BdvPtr Armory::Bridge::setupClientConnection( std::shared_ptr peers, - const std::string& ip, const std::string& port, bool oneWayAuth, + const std::string& ip, Network::port_t port, bool oneWayAuth, const std::function& presentPubKeyFunc, std::shared_ptr cbPtr) { @@ -676,14 +689,16 @@ void AutomationContext::automateSatoshi() } std::vector args{ - std::format("--datadir={}", datadir.string()), std::format("--rpcauth={}:{}${}", rpcLogin_, salt, saltedPass.toHexStr()), {"--disablewallet"} }; if (Config::BitcoinSettings::getMode() == Config::NETWORK_MODE_TESTNET) { args.emplace_back("--testnet"); } - auto result = spawnProcess(satoshiBin_, args, {}, false); + auto result = spawnProcess(satoshiBin_, + {{"--datadir", datadir}}, + args, {}, false + ); nodeInstance_ = result.first; if (!nodeInstance_.isValid()) { throw std::runtime_error(std::format( @@ -767,16 +782,12 @@ void AutomationContext::automateDb() //generate random db port & set it dbPort_ = (rand() % 10000) + 50000; - auto portStr = std::to_string(dbPort_); - Armory::Config::NetworkSettings::setDbPort(portStr); + Armory::Config::NetworkSettings::setDbPort(dbPort_); //args std::vector args{ { "--ephemeral" }, - std::format("--armorydb-port={}", portStr), - std::format("--dbdir={}", dbDir_.string()), - std::format("--datadir={}", Config::getDataDir().string()), - std::format("--satoshi-datadir={}", satoshiDir_.string()), + std::format("--armorydb-port={}", dbPort_), std::format("--satoshi-port={}", Config::NetworkSettings::btcPort()), std::format("--satoshirpc-port={}", Config::NetworkSettings::rpcPort()) }; @@ -806,7 +817,12 @@ void AutomationContext::automateDb() envvars.emplace("CORERPCPASS", rpcPass_); } - auto result = spawnProcess(armoryDbPath, args, envvars, true); + auto result = spawnProcess(armoryDbPath, { + {"--dbdir", dbDir_}, + {"--datadir", Config::getDataDir()}, + {"--satoshi-datadir", satoshiDir_} + }, args, envvars, true + ); dbInstance_ = result.first; if (!dbInstance_.isValid()) { throw std::runtime_error(std::format( @@ -817,7 +833,7 @@ void AutomationContext::automateDb() //set db pubkey auto serverKey = NetworkPeers::PeerKey::fromHumanReadable(result.second); - peers_->addPeer(serverKey, {std::format("127.0.0.1:{}", portStr)}, {}); + peers_->addPeer(serverKey, {std::format("127.0.0.1:{}", dbPort_)}, {}); } //// @@ -836,9 +852,8 @@ void AutomationContext::cleanupDb() //create bdv object auto callback = std::make_shared(); - auto port = std::to_string(dbPort_); auto bdvPtr = setupClientConnection(peers_, - "127.0.0.1", port, + "127.0.0.1", dbPort_, false, nullptr, callback); if (bdvPtr == nullptr) { throw std::runtime_error("automatedDb connection failed"); diff --git a/cppForSwig/BridgeAPI/DBSetup.h b/cppForSwig/BridgeAPI/DBSetup.h index 2a6c574fb..b1c537d45 100644 --- a/cppForSwig/BridgeAPI/DBSetup.h +++ b/cppForSwig/BridgeAPI/DBSetup.h @@ -110,7 +110,7 @@ namespace Armory BdvPtr setupClientConnection( std::shared_ptr, - const std::string&, const std::string&, bool, + const std::string&, uint16_t, bool, const std::function&, std::shared_ptr ); diff --git a/cppForSwig/BridgeAPI/ProtoCommandParser.cpp b/cppForSwig/BridgeAPI/ProtoCommandParser.cpp index b615aa912..310020b82 100644 --- a/cppForSwig/BridgeAPI/ProtoCommandParser.cpp +++ b/cppForSwig/BridgeAPI/ProtoCommandParser.cpp @@ -88,7 +88,7 @@ namespace { auto connectReq = request.getConnectToIp(); std::string ip = connectReq.getIp(); - std::string port = connectReq.getPort(); + Network::port_t port = connectReq.getPort(); std::string callbackId = connectReq.getCallbackId(); std::thread thr([bridge, ip, port, callbackId, referenceId]{ @@ -223,10 +223,10 @@ namespace auto fromBridge = message.initRoot(); auto reply = fromBridge.initReply(); reply.setReferenceId(referenceId); - + try { - auto datadir = std::filesystem::path( - satoshiRequest.getValidateDir()); + auto datadir = std::filesystem::path{ + std::string{satoshiRequest.getValidateDir()}}; auto validationResult = Node::Core::validateDatadir(datadir); @@ -254,8 +254,8 @@ namespace reply.setReferenceId(referenceId); try { - auto binPath = std::filesystem::path( - satoshiRequest.getValidateBin()); + auto binPath = std::filesystem::path{ + std::string{satoshiRequest.getValidateBin()}}; auto validationResult = Node::Core::validateBinary(binPath); auto setupReply = reply.initSetup(); @@ -280,15 +280,15 @@ namespace auto ctxReq = request.getInitAutomationContext(); std::filesystem::path dbDir; if (ctxReq.hasDbDir()) { - dbDir = std::filesystem::path{ctxReq.getDbDir()}; + dbDir = std::filesystem::path{std::string{ctxReq.getDbDir()}}; } std::filesystem::path satoshiDir; if (ctxReq.hasSatoshiDir()) { - satoshiDir = std::filesystem::path{ctxReq.getSatoshiDir()}; + satoshiDir = std::filesystem::path{std::string{ctxReq.getSatoshiDir()}}; } std::filesystem::path satoshiBin; if (ctxReq.hasSatoshiBin()) { - satoshiBin = std::filesystem::path{ctxReq.getSatoshiBin()}; + satoshiBin = std::filesystem::path{std::string{ctxReq.getSatoshiBin()}}; } bool automateDb = false; @@ -489,7 +489,7 @@ namespace case WalletManagerRequest::MIGRATE_WALLET: { auto migrateReq = request.getMigrateWallet(); - const std::filesystem::path walletPath(std::string{migrateReq.getWalletPath()}); + const std::filesystem::path walletPath{std::string{migrateReq.getWalletPath()}}; const std::string callbackId(migrateReq.getCallbackId()); bridge->migrateWallet(walletPath, callbackId, referenceId); break; @@ -1457,7 +1457,7 @@ namespace case UtilsRequest::IMPORT_WALLET: { - std::filesystem::path importPath(std::string{request.getImportWallet()}); + std::filesystem::path importPath{std::string{request.getImportWallet()}}; bridge->importWallet(importPath, referenceId); break; } diff --git a/cppForSwig/Network/SocketObject.cpp b/cppForSwig/Network/SocketObject.cpp index 12765f810..7921932ad 100644 --- a/cppForSwig/Network/SocketObject.cpp +++ b/cppForSwig/Network/SocketObject.cpp @@ -31,16 +31,16 @@ char *gai_strerrorA(int) { return nullptr; } // /////////////////////////////////////////////////////////////////////////////// SocketPrototype::SocketPrototype() : - addr_(""), port_("") + addr_(""), port_(UINT16_MAX) {} //////// SocketPrototype::SocketPrototype(const std::string& addr, - const std::string& port, bool doInit) : + port_t port, bool doInit) : addr_(addr), port_(port) { - if (addr.empty() || port.empty()) { - throw std::runtime_error("empty addr/port"); + if (addr.empty() || port == 0 || port == UINT16_MAX) { + throw std::runtime_error("invalid addr/port"); } if (doInit) { @@ -64,12 +64,6 @@ const std::string& SocketPrototype::getAddrStr() const return addr_; } -//////// -const std::string& SocketPrototype::getPortStr() const -{ - return port_; -} - //////// bool SocketPrototype::running() const { @@ -97,7 +91,8 @@ void SocketPrototype::init() auto& addrstr = addr_; #endif - getaddrinfo(addrstr.c_str(), port_.c_str(), &hints, &result); + const auto& portStr = std::to_string(port_); + getaddrinfo(addrstr.c_str(), portStr.c_str(), &hints, &result); for (auto ptr = result; ptr != nullptr; ptr = ptr->ai_next) { if (ptr->ai_family == AF_INET) { memcpy(&serv_addr_, ptr->ai_addr, sizeof(sockaddr_in)); @@ -146,7 +141,7 @@ void SocketPrototype::closeSocket(SOCKET& sockfd) } /////////////////////////////////////////////////////////////////////////////// -bool SocketPrototype::testConnection(void) +bool SocketPrototype::testConnection() { try { auto sockfd = openSocket(true); @@ -177,7 +172,7 @@ void SocketPrototype::setBlocking(SOCKET sock, bool setblocking) return; } - flags = setblocking ? (flags&~O_NONBLOCK) : (flags | O_NONBLOCK); + flags = setblocking ? (flags & ~O_NONBLOCK) : (flags | O_NONBLOCK); int rt = fcntl(sock, F_SETFL, flags); if (rt != 0) { std::cout << "fcntl returned " << rt << std::endl; @@ -275,8 +270,7 @@ void SocketPrototype::listen(AcceptCallback callback, SOCKET& sockfd) //// PersistentSocket // /////////////////////////////////////////////////////////////////////////////// -PersistentSocket::PersistentSocket(const std::string& addr, - const std::string& port) : +PersistentSocket::PersistentSocket(const std::string& addr, port_t port) : SocketPrototype(addr, port) { shutdownProm_ = std::make_unique>(); @@ -693,7 +687,8 @@ void PersistentSocket::cleanUpPipes() /////////////////////////////////////////////////////////////////////////////// bool PersistentSocket::openSocket(bool blocking) { - if (!addr_.empty() && !port_.empty() && readQueue_.isValid()) { + if (!addr_.empty() && port_ != 0 && port_ != UINT16_MAX && + readQueue_.isValid()) { sockfd_ = SocketPrototype::openSocket(blocking); } return isValid(); @@ -769,7 +764,6 @@ void PersistentSocket::shutdown() shutdownProm_->set_value(true); } -/////////////////////////////////////////////////////////////////////////////// void PersistentSocket::blockUntilClosed() const { auto futCopy = shutdownFut_; @@ -781,7 +775,7 @@ void PersistentSocket::blockUntilClosed() const //// SimpleSocket // /////////////////////////////////////////////////////////////////////////////// -SimpleSocket::SimpleSocket(const std::string& addr, const std::string& port) : +SimpleSocket::SimpleSocket(const std::string& addr, port_t port) : SocketPrototype(addr, port) {} @@ -953,8 +947,7 @@ bool SimpleSocket::connectToRemote() return sockfd_ != SOCK_MAX; } -/////////////////////////////////////////////////////////////////////////////// -bool SimpleSocket::checkSocket(const std::string& ip, const std::string& port) +bool SimpleSocket::checkSocket(const std::string& ip, port_t port) { SimpleSocket testSock(ip, port); return testSock.testConnection(); @@ -965,7 +958,7 @@ bool SimpleSocket::checkSocket(const std::string& ip, const std::string& port) //// ListenServer // /////////////////////////////////////////////////////////////////////////////// -ListenServer::ListenServer(const std::string& addr, const std::string& port) +ListenServer::ListenServer(const std::string& addr, port_t port) { listenSocket_ = std::make_unique(addr, port); listenSocket_->verbose_ = false; diff --git a/cppForSwig/Network/SocketObject.h b/cppForSwig/Network/SocketObject.h index 3800379ff..49c1ae128 100644 --- a/cppForSwig/Network/SocketObject.h +++ b/cppForSwig/Network/SocketObject.h @@ -33,6 +33,8 @@ namespace Armory { namespace Network { + using port_t = uint16_t; + ////////////////////////////////////////////////////////////////////////// struct CallbackReturn { @@ -103,8 +105,7 @@ namespace Armory struct sockaddr serv_addr_; const std::string addr_; - const std::string port_; - + const port_t port_; bool verbose_ = true; private: @@ -114,15 +115,15 @@ namespace Armory SocketPrototype(void); void setBlocking(SOCKET, bool); - void listen(AcceptCallback, SOCKET& sockfd); + void listen(AcceptCallback, SOCKET&); public: - SocketPrototype(const std::string& addr, const std::string& port, bool init = true); + SocketPrototype(const std::string&, port_t, bool = true); virtual ~SocketPrototype(void) = 0; virtual bool testConnection(void); bool isBlocking(void) const; - SOCKET openSocket(bool blocking); + SOCKET openSocket(bool); static void closeSocket(SOCKET&); virtual void pushPayload( @@ -132,7 +133,6 @@ namespace Armory virtual SocketType type(void) const = 0; const std::string& getAddrStr(void) const; - const std::string& getPortStr(void) const; //override me virtual bool running(void) const; @@ -147,7 +147,7 @@ namespace Armory int writeToSocket(std::vector&); public: - SimpleSocket(const std::string& addr, const std::string& port); + SimpleSocket(const std::string&, port_t); SimpleSocket(SOCKET); ~SimpleSocket(void); @@ -162,7 +162,7 @@ namespace Armory bool connectToRemote(void) override; // - static bool checkSocket(const std::string& ip, const std::string& port); + static bool checkSocket(const std::string&, port_t); }; class PersistentSocket : public SocketPrototype @@ -208,15 +208,15 @@ namespace Armory void queuePayloadForWrite(std::vector&); public: - PersistentSocket(const std::string& addr, const std::string& port); + PersistentSocket(const std::string&, port_t); PersistentSocket(SOCKET); ~PersistentSocket(void); - void shutdown(); + void shutdown(void); bool openSocket(bool); int getSocketName(struct sockaddr& ); int getPeerName(struct sockaddr&); - bool connectToRemote(void); + bool connectToRemote(void) override; bool isValid(void) const; bool testConnection(void); void blockUntilClosed(void) const; @@ -253,7 +253,7 @@ namespace Armory ListenServer(const ListenServer&) = delete; public: - ListenServer(const std::string& addr, const std::string& port); + ListenServer(const std::string&, port_t); ~ListenServer(void); void start(ReadCallback); diff --git a/cppForSwig/Network/StringSockets.cpp b/cppForSwig/Network/StringSockets.cpp index bb5a623d0..f354e7972 100644 --- a/cppForSwig/Network/StringSockets.cpp +++ b/cppForSwig/Network/StringSockets.cpp @@ -15,7 +15,7 @@ using namespace Armory::Network; /////////////////////////////////////////////////////////////////////////////// // HttpSocket -HttpSocket::HttpSocket(const std::string& addr, const std::string& port) : +HttpSocket::HttpSocket(const std::string& addr, port_t port) : SimpleSocket(addr, port) { messageWithPrecacheHeaders_ = make_unique(getAddrStr()); diff --git a/cppForSwig/Network/StringSockets.h b/cppForSwig/Network/StringSockets.h index 7b4b41cd7..a4cf4f4ba 100644 --- a/cppForSwig/Network/StringSockets.h +++ b/cppForSwig/Network/StringSockets.h @@ -90,7 +90,7 @@ namespace Armory std::string getHttpPayload(const char*, size_t); public: - HttpSocket(const std::string& addr, const std::string& port); + HttpSocket(const std::string&, port_t); static size_t getHttpBodyOffset(const char*, size_t); SocketType type(void) const override; diff --git a/cppForSwig/Network/WebSocketClient.cpp b/cppForSwig/Network/WebSocketClient.cpp index 94c982486..6453e2e6f 100755 --- a/cppForSwig/Network/WebSocketClient.cpp +++ b/cppForSwig/Network/WebSocketClient.cpp @@ -37,12 +37,12 @@ static struct lws_protocols protocols[] = { //////////////////////////////////////////////////////////////////////////////// // WebSocketClient -WebSocketClient::WebSocketClient(const std::string& addr, - const std::string& port, +WebSocketClient::WebSocketClient(const std::string& addr, port_t port, std::shared_ptr peers, bool oneWayAuth, std::shared_ptr cbPtr) : SocketPrototype(addr, port, false), - servName_(addr_ + ":" + port_), callbackPtr_(cbPtr), peerStore_(peers) + servName_(std::format("{}:{}", addr_, port_)), + callbackPtr_(cbPtr), peerStore_(peers) { count_.store(0, std::memory_order_relaxed); contextPtr_.store(0, std::memory_order_release); @@ -189,12 +189,7 @@ struct lws_context* WebSocketClient::init() //connect to server struct lws_client_connect_info i; memset(&i, 0, sizeof(i)); - - int port = std::stoi(port_); - if (port == 0) { - port = WEBSOCKET_PORT; - } - i.port = port; + i.port = port_ == UINT16_MAX ? WEBSOCKET_PORT : port_; const char *prot, *p; char path[300]; @@ -629,8 +624,7 @@ void WebSocketClient::addPublicKey(const SecureBinaryData& pubkey, bool oneWay) NetworkPeers::PeerType::ServerOneWay : NetworkPeers::PeerType::ServerTwoWay }; - const std::string addrPort{ addr_ + ":" + port_ }; - peerStore_->addPeer(serverKey, {addrPort}, {}); + peerStore_->addPeer(serverKey, {std::format("{}:{}", addr_, port_)}, {}); } void WebSocketClient::setPubkeyPromptLambda( diff --git a/cppForSwig/Network/WebSocketClient.h b/cppForSwig/Network/WebSocketClient.h index 275e5f877..cce36318b 100644 --- a/cppForSwig/Network/WebSocketClient.h +++ b/cppForSwig/Network/WebSocketClient.h @@ -140,7 +140,7 @@ namespace Armory void cleanup(void); public: - WebSocketClient(const std::string& addr, const std::string& port, + WebSocketClient(const std::string&, port_t, std::shared_ptr, bool, std::shared_ptr); ~WebSocketClient(void); diff --git a/cppForSwig/Node/BitcoinP2P.cpp b/cppForSwig/Node/BitcoinP2P.cpp index 36bf6eb5c..96530c2b2 100644 --- a/cppForSwig/Node/BitcoinP2P.cpp +++ b/cppForSwig/Node/BitcoinP2P.cpp @@ -1280,7 +1280,7 @@ void P2P::Iface::requestTx(InvVector invVec) //////////////////////////////////////////////////////////////////////////////// // Peer P2P::Peer::Peer( - const std::string& addrV4, const std::string& port, + const std::string& addrV4, Network::port_t port, uint32_t magicword, bool watcher) : Iface(magicword, watcher), addr_(addrV4), port_(port) { @@ -1679,7 +1679,7 @@ bool P2P::Peer::connected() const //////////////////////////////////////////////////////////////////////////////// // Socket P2P::Socket::Socket( - const std::string& addr, const std::string& port, + const std::string& addr, Network::port_t port, std::shared_ptr>> readStack) : PersistentSocket(addr, port), readDataStack_(readStack) {} diff --git a/cppForSwig/Node/BitcoinP2P.h b/cppForSwig/Node/BitcoinP2P.h index 51b130524..d3948865c 100644 --- a/cppForSwig/Node/BitcoinP2P.h +++ b/cppForSwig/Node/BitcoinP2P.h @@ -371,7 +371,7 @@ namespace Node std::vector>> readDataStack_; public: - Socket(const std::string&, const std::string&, + Socket(const std::string&, Armory::Network::port_t, std::shared_ptr>>); SocketType type(void) const override; @@ -443,7 +443,7 @@ namespace Node { private: const std::string addr_; - const std::string port_; + Armory::Network::port_t port_; struct sockaddr node_addr_; std::unique_ptr socket_; @@ -483,7 +483,7 @@ namespace Node void sendMessage(std::vector>); public: - Peer(const std::string&, const std::string&, + Peer(const std::string&, Armory::Network::port_t, MagicWordType, bool); ~Peer(void); diff --git a/cppForSwig/Server.cpp b/cppForSwig/Server.cpp index eea41fcee..7fd81f3c2 100755 --- a/cppForSwig/Server.cpp +++ b/cppForSwig/Server.cpp @@ -305,10 +305,8 @@ void WebSocketServer::start(std::shared_ptr bdm, bool async) [instance]{ instance->clientInterruptThread(); })); } - auto port = stoi(Config::NetworkSettings::dbPort()); - if (port == 0) { - port = WEBSOCKET_PORT; - } + auto port = Config::NetworkSettings::dbPort() == UINT16_MAX ? + WEBSOCKET_PORT : Config::NetworkSettings::dbPort(); //run service thread if (async) { diff --git a/cppForSwig/Utils/ArmoryConfig.cpp b/cppForSwig/Utils/ArmoryConfig.cpp index 241d22efb..e6fb3d6b7 100755 --- a/cppForSwig/Utils/ArmoryConfig.cpp +++ b/cppForSwig/Utils/ArmoryConfig.cpp @@ -23,7 +23,6 @@ #include #include -namespace fs = std::filesystem; using namespace std::literals::string_view_literals; using namespace Armory; using namespace Armory::Config; @@ -127,7 +126,7 @@ void Armory::Config::printHelp(void) } //////////////////////////////////////////////////////////////////////////////// -const fs::path& Armory::Config::getDataDir() +const std::filesystem::path& Armory::Config::getDataDir() { return BaseSettings::dataDir_; } @@ -140,9 +139,9 @@ void Armory::Config::parseArgs(int argc, char* argv[], ProcessType procType) for (int i=1; i args; for (const auto& line : lines) { if (line == ("--help")) { - Armory::Config::printHelp(); + printHelp(); exit(0); } @@ -190,7 +189,7 @@ void Armory::Config::parseArgs( BaseSettings::detectDataDir(args); //get config file - auto configPath = fs::path(Armory::Config::getDataDir()) / "armorydb.conf"; + auto configPath = std::filesystem::path(getDataDir()) / "armorydb.conf"; if (FileUtils::pathExists(configPath, 2)) { Config::File cf(configPath); auto mapIter = cf.keyvalMap_.find("datadir"); @@ -222,7 +221,6 @@ void Armory::Config::parseArgs( void Armory::Config::reset() { std::unique_lock lock(BaseSettings::configMutex_); - NetworkSettings::reset(); Pathing::reset(); DBSettings::reset(); @@ -234,7 +232,8 @@ void Armory::Config::reset() // SettingsUtils // //////////////////////////////////////////////////////////////////////////////// -std::vector SettingsUtils::getLines(const fs::path& path) +std::vector SettingsUtils::getLines( + const std::filesystem::path& path) { std::vector output; std::fstream inStream(path, std::ios_base::in); @@ -338,76 +337,15 @@ std::string_view SettingsUtils::stripQuotes(const std::string_view& input) return std::string_view(input.begin() + start, len); } -//////////////////////////////////////////////////////////////////////////////// -bool SettingsUtils::testConnection(const std::string& ip, const std::string& port) -{ - Network::SimpleSocket testSock(ip, port); - return testSock.testConnection(); -} - -//////////////////////////////////////////////////////////////////////////////// -std::string SettingsUtils::getPortFromCookie(const std::string& datadir) -{ - //check for cookie file - auto cookie_path = fs::path(datadir) / ".cookie_"; - auto lines = SettingsUtils::getLines(cookie_path); - if (lines.size() != 2) { - return {}; - } - return lines[1]; -} - -//////////////////////////////////////////////////////////////////////////////// -std::string SettingsUtils::hasLocalDB( - const std::string& datadir, const std::string& port) -{ - //check db on provided port - if (SettingsUtils::testConnection("127.0.0.1", port)) { - return port; - } - - //check db on default port - std::string defaultPort; - switch (BitcoinSettings::getMode()) - { - case NETWORK_MODE_TESTNET: - defaultPort = std::to_string(LISTEN_PORT_TESTNET); - break; - - case NETWORK_MODE_REGTEST: - defaultPort = std::to_string(LISTEN_PORT_REGTEST); - break; - - default: - defaultPort = std::to_string(LISTEN_PORT_MAINNET); - } - - if (SettingsUtils::testConnection("127.0.0.1", defaultPort)) { - return defaultPort; - } - - //check for cookie file - auto cookie_port = getPortFromCookie(datadir); - if (cookie_port.empty()) { - return {}; - } - - if (SettingsUtils::testConnection("127.0.0.1", cookie_port)) { - return cookie_port; - } - return {}; -} - //////////////////////////////////////////////////////////////////////////////// // // BaseSettings // //////////////////////////////////////////////////////////////////////////////// std::mutex BaseSettings::configMutex_; -fs::path BaseSettings::dataDir_; +std::filesystem::path BaseSettings::dataDir_; unsigned BaseSettings::initCount_ = 0; -//////////////////////////////////////////////////////////////////////////////// void BaseSettings::detectDataDir(std::map& args) { //figure out the datadir @@ -446,7 +384,7 @@ void BaseSettings::detectDataDir(std::map& args) isAuto = true; } - dataDir_ = fs::absolute(dataDir_); + dataDir_ = std::filesystem::absolute(dataDir_); if (!isAuto) { return; } @@ -455,7 +393,6 @@ void BaseSettings::detectDataDir(std::map& args) FileUtils::createDirectory(dataDir_); } -//////////////////////////////////////////////////////////////////////////////// void BaseSettings::reset() { dataDir_.clear(); @@ -745,16 +682,13 @@ void DBSettings::reset() // NetworkSettings // //////////////////////////////////////////////////////////////////////////////// -bool NetworkSettings::customDbPort_ = false; -bool NetworkSettings::customBtcPort_ = false; - NetworkSettings::NodePair NetworkSettings::bitcoinNodes_; NetworkSettings::RpcPtr NetworkSettings::rpcNode_; -std::string NetworkSettings::btcPort_; -std::string NetworkSettings::dbPort_; +Network::port_t NetworkSettings::btcPort_ = UINT16_MAX; +Network::port_t NetworkSettings::dbPort_ = UINT16_MAX; +Network::port_t NetworkSettings::rpcPort_ = UINT16_MAX; std::string NetworkSettings::dbIP_; -std::string NetworkSettings::rpcPort_; bool NetworkSettings::ephemeralPeers_; bool NetworkSettings::oneWayAuth_ = false; @@ -767,19 +701,47 @@ void NetworkSettings::processArgs( const std::map& args, ProcessType procType) { + //db port auto iter = args.find("armorydb-port"); if (iter != args.end()) { - dbPort_ = SettingsUtils::stripQuotes(iter->second); - int portInt = std::stoi(dbPort_); + auto portSv = SettingsUtils::stripQuotes(iter->second); + auto [ptr, ec] = std::from_chars(portSv.data(), + portSv.data() + portSv.size(), dbPort_); + + if (ec != std::errc{} || dbPort_ == 0 || dbPort_ == UINT16_MAX) { + std::cout << "Invalid listen port, falling back to default: " << + portSv << ", " << dbPort_ << std::endl; + dbPort_ = UINT16_MAX; + } + } - if (portInt < 1 || portInt > 65535) { - std::cout << "Invalid listen port, falling back to default" << std::endl; - dbPort_ = ""; - } else { - customDbPort_ = true; + //node port + iter = args.find("satoshi-port"); + if (iter != args.end()) { + auto portSv = SettingsUtils::stripQuotes(iter->second); + auto [ptr, ec] = std::from_chars(portSv.data(), + portSv.data() + portSv.size(), btcPort_); + + if (ec != std::errc{} || btcPort_ == 0 || btcPort_ == UINT16_MAX) { + std::cout << "Invalid node port, falling back to default" << std::endl; + btcPort_ = UINT16_MAX; } } + //rpc port + iter = args.find("satoshirpc-port"); + if (iter != args.end()) { + auto portSv = SettingsUtils::stripQuotes(iter->second); + auto [ptr, ec] = std::from_chars(portSv.data(), + portSv.data() + portSv.size(), rpcPort_); + + if (ec != std::errc{} || rpcPort_ == 0 || rpcPort_ == UINT16_MAX) { + std::cout << "Invalid rpc port, falling back to default" << std::endl; + rpcPort_ = UINT16_MAX; + } + } + + //db IP iter = args.find("armorydb-ip"); if (iter != args.end()) { dbIP_ = SettingsUtils::stripQuotes(iter->second); @@ -787,12 +749,6 @@ void NetworkSettings::processArgs( dbIP_ = "127.0.0.1"; } - iter = args.find("satoshi-port"); - if (iter != args.end()) { - btcPort_ = SettingsUtils::stripQuotes(iter->second); - customBtcPort_ = true; - } - //network type iter = args.find("testnet"); if (iter != args.end()) { @@ -806,24 +762,6 @@ void NetworkSettings::processArgs( } } - //rpc port - iter = args.find("satoshirpc-port"); - if (iter != args.end()) { - auto value = SettingsUtils::stripQuotes(iter->second); - int portInt; - - try { - std::from_chars(value.begin(), value.end(), portInt); - if (portInt < 1 || portInt > 65535) { - std::cout << "Invalid satoshi rpc port, falling back to default" << std::endl; - } else { - rpcPort_ = value; - } - } catch (const std::exception&) { - std::cout << "satoshi rpc port is not a number, falling back to default" << std::endl; - } - } - //public iter = args.find("public"); if (iter != args.end()) { @@ -864,42 +802,48 @@ void NetworkSettings::selectNetwork(NETWORK_MODE mode) { case NETWORK_MODE_MAINNET: { - rpcPort_ = std::to_string(RPC_PORT_MAINNET); + if (rpcPort_ == UINT16_MAX) { + rpcPort_ = RPC_PORT_MAINNET; + } - if (!customDbPort_) { - dbPort_ = std::to_string(LISTEN_PORT_MAINNET); + if (dbPort_ == UINT16_MAX) { + dbPort_ = LISTEN_PORT_MAINNET; } - if (!customBtcPort_) { - btcPort_ = std::to_string(NODE_PORT_MAINNET); + if (btcPort_ == UINT16_MAX) { + btcPort_ = NODE_PORT_MAINNET; } break; } case NETWORK_MODE_TESTNET: { - rpcPort_ = std::to_string(RPC_PORT_TESTNET); + if (rpcPort_ == UINT16_MAX) { + rpcPort_ = RPC_PORT_TESTNET; + } - if (!customDbPort_) { - dbPort_ = std::to_string(LISTEN_PORT_TESTNET); + if (dbPort_ == UINT16_MAX) { + dbPort_ = LISTEN_PORT_TESTNET; } - if (!customBtcPort_) { - btcPort_ = std::to_string(NODE_PORT_TESTNET); + if (btcPort_ == UINT16_MAX) { + btcPort_ = NODE_PORT_TESTNET; } break; } case NETWORK_MODE_REGTEST: { - rpcPort_ = std::to_string(RPC_PORT_REGTEST); + if (rpcPort_ == UINT16_MAX) { + rpcPort_ = RPC_PORT_REGTEST; + } - if (!customDbPort_) { - dbPort_ = std::to_string(LISTEN_PORT_REGTEST); + if (dbPort_ == UINT16_MAX) { + dbPort_ = LISTEN_PORT_REGTEST; } - if (!customBtcPort_) { - btcPort_ = std::to_string(NODE_PORT_REGTEST); + if (btcPort_ == UINT16_MAX) { + btcPort_ = NODE_PORT_REGTEST; } break; } @@ -910,43 +854,30 @@ void NetworkSettings::selectNetwork(NETWORK_MODE mode) } //////////////////////////////////////////////////////////////////////////////// -void NetworkSettings::setDbPort(const std::string& port) +void NetworkSettings::setDbPort(Network::port_t port) { dbPort_ = port; } -//////////////////////////////////////////////////////////////////////////////// -const std::string& NetworkSettings::btcPort() +//// +Network::port_t NetworkSettings::btcPort() { return btcPort_; } -std::wstring NetworkSettings::btcPortW() -{ - return std::to_wstring(std::stoi(btcPort_)); -} - -//////////////////////////////////////////////////////////////////////////////// -const std::string& NetworkSettings::dbPort() +Network::port_t NetworkSettings::dbPort() { return dbPort_; } -//////////////////////////////////////////////////////////////////////////////// -const std::string& NetworkSettings::dbIP() -{ - return dbIP_; -} - -//////////////////////////////////////////////////////////////////////////////// -const std::string& NetworkSettings::rpcPort() +Network::port_t NetworkSettings::rpcPort() { return rpcPort_; } -std::wstring NetworkSettings::rpcPortW() +const std::string& NetworkSettings::dbIP() { - return std::to_wstring(std::stoi(rpcPort_)); + return dbIP_; } //////////////////////////////////////////////////////////////////////////////// @@ -1004,17 +935,14 @@ NetworkSettings::RpcPtr NetworkSettings::rpcNode() //////////////////////////////////////////////////////////////////////////////// void NetworkSettings::reset() { - customDbPort_ = false; - customBtcPort_ = false; - bitcoinNodes_.first.reset(); bitcoinNodes_.second.reset(); rpcNode_.reset(); - btcPort_.clear(); - dbPort_.clear(); + btcPort_ = UINT16_MAX; + dbPort_ = UINT16_MAX; + rpcPort_ = UINT16_MAX; dbIP_.clear(); - rpcPort_.clear(); ephemeralPeers_ = false; oneWayAuth_ = false; @@ -1026,11 +954,10 @@ void NetworkSettings::reset() // Pathing // //////////////////////////////////////////////////////////////////////////////// -fs::path Pathing::blkFilePath_; -fs::path Pathing::dbDir_; -fs::path Pathing::own_; +std::filesystem::path Pathing::blkFilePath_; +std::filesystem::path Pathing::dbDir_; +std::filesystem::path Pathing::own_; -//////////////////////////////////////////////////////////////////////////////// void Pathing::processArgs(const std::map& args, ProcessType procType) { @@ -1067,17 +994,17 @@ void Pathing::processArgs(const std::map& args, } //expand paths if necessary - dbDir_ = fs::absolute(dbDir_); - blkFilePath_ = fs::absolute(blkFilePath_); + dbDir_ = std::filesystem::absolute(dbDir_); + blkFilePath_ = std::filesystem::absolute(blkFilePath_); //check block file path ends in "blocks" if (blkFilePath_.filename() != "blocks") { - blkFilePath_ = fs::path(blkFilePath_) / fs::path("blocks"); + blkFilePath_ = blkFilePath_ / "blocks"; } //test all paths - if (!FileUtils::pathExists(Armory::Config::getDataDir(), 6)) { - throw DbErrorMsg({Armory::Config::getDataDir().string() + + if (!FileUtils::pathExists(getDataDir(), 6)) { + throw DbErrorMsg({getDataDir().string() + " is not a valid datadir path"}); } @@ -1094,7 +1021,7 @@ void Pathing::processArgs(const std::map& args, //create dbdir if set automatically if (autoDbDir) { if (!FileUtils::pathExists(dbDir_, 0)) { - fs::create_directory(dbDir_); + std::filesystem::create_directory(dbDir_); } } @@ -1125,25 +1052,25 @@ void Pathing::reset() } //////////////////////////////////////////////////////////////////////////////// -fs::path Pathing::logFilePath(const std::string& logName) +std::filesystem::path Pathing::logFilePath(const std::string& logName) { - return fs::path(getDataDir()) / fs::path(logName + ".txt"); + return getDataDir() / std::format("{}.txt", logName); } //// -const fs::path& Pathing::blkFilePath() +const std::filesystem::path& Pathing::blkFilePath() { return blkFilePath_; } //// -const fs::path& Pathing::dbDir() +const std::filesystem::path& Pathing::dbDir() { return dbDir_; } //// -const fs::path& Pathing::runningDir() +const std::filesystem::path& Pathing::runningDir() { return own_; } @@ -1153,7 +1080,7 @@ const fs::path& Pathing::runningDir() // ConfigFile // //////////////////////////////////////////////////////////////////////////////// -Config::File::File(const fs::path& path) +Config::File::File(const std::filesystem::path& path) { auto lines = SettingsUtils::getLines(path); for (auto& line : lines) { @@ -1194,7 +1121,7 @@ std::vector Config::File::fleshOutArgs( auto keyValMap = SettingsUtils::getKeyValsFromLines(arg_minus_1, '='); //complete config file path - auto configFilePath = fs::path(MAINNET_DEFAULT_DATADIR); + auto configFilePath = std::filesystem::path(MAINNET_DEFAULT_DATADIR); if (keyValMap.find("--testnet") != keyValMap.end()) { configFilePath = TESTNET_DEFAULT_DATADIR; } else if (keyValMap.find("--regtest") != keyValMap.end()) { @@ -1205,7 +1132,7 @@ std::vector Config::File::fleshOutArgs( if (datadir_iter != keyValMap.end() && !datadir_iter->second.empty()) { configFilePath = datadir_iter->second; } - configFilePath = fs::absolute(configFilePath / path); + configFilePath = std::filesystem::absolute(configFilePath / path); //process config file Config::File cfile(configFilePath); diff --git a/cppForSwig/Utils/ArmoryConfig.h b/cppForSwig/Utils/ArmoryConfig.h index e98931ba3..f7d5e1dab 100755 --- a/cppForSwig/Utils/ArmoryConfig.h +++ b/cppForSwig/Utils/ArmoryConfig.h @@ -76,10 +76,6 @@ namespace Armory std::string_view stripQuotes(const std::string_view& input); std::vector keyValToArgv( const std::map&); - - bool testConnection(const std::string&, const std::string&); - std::string getPortFromCookie(const std::string&); - std::string hasLocalDB(const std::string&, const std::string&); }; ////////////////////////////////////////////////////////////////////////// @@ -180,13 +176,10 @@ namespace Armory static NodePair bitcoinNodes_; static RpcPtr rpcNode_; - static std::string btcPort_; - static std::string dbPort_; + static uint16_t btcPort_; + static uint16_t dbPort_; + static uint16_t rpcPort_; static std::string dbIP_; - static std::string rpcPort_; - - static bool customDbPort_; - static bool customBtcPort_; static bool ephemeralPeers_; static bool oneWayAuth_; @@ -205,16 +198,14 @@ namespace Armory public: static void selectNetwork(NETWORK_MODE); - static const std::string& btcPort(void); - static std::wstring btcPortW(void); - static const std::string& dbPort(void); + static uint16_t btcPort(void); + static uint16_t dbPort(void); + static uint16_t rpcPort(void); static const std::string& dbIP(void); - static const std::string& rpcPort(void); - static std::wstring rpcPortW(void); static const NodePair& bitcoinNodes(void); static RpcPtr rpcNode(void); - static void setDbPort(const std::string&); + static void setDbPort(uint16_t); static bool ephemeralPeers(void) { return ephemeralPeers_; } static bool oneWayAuth(void) { return oneWayAuth_; } diff --git a/cppForSwig/Utils/BtcUtils.cpp b/cppForSwig/Utils/BtcUtils.cpp index 5e4f7a45e..e8ad8aa2b 100644 --- a/cppForSwig/Utils/BtcUtils.cpp +++ b/cppForSwig/Utils/BtcUtils.cpp @@ -1465,7 +1465,7 @@ std::string BtcUtils::base64_decode(const std::string& in) break; } throw std::runtime_error( - std::format("invalid b64 character: {}", val8)); + std::format("invalid b64 character: \"{}\"", val8)); } uint32_t bits = iter->second << (26 - (6 * (y % 4))); diff --git a/cppForSwig/capnp/Bridge.capnp b/cppForSwig/capnp/Bridge.capnp index e38402bfd..888da7e7d 100644 --- a/cppForSwig/capnp/Bridge.capnp +++ b/cppForSwig/capnp/Bridge.capnp @@ -241,7 +241,7 @@ struct NotificationReply { struct DbSetupRequest { struct IpRequest { ip @0 : Text; - port @1 : Text; + port @1 : UInt16; callbackId @2 : Text; } diff --git a/cppForSwig/gtest/BridgeTests.cpp b/cppForSwig/gtest/BridgeTests.cpp index 20812a66f..e8273ada7 100644 --- a/cppForSwig/gtest/BridgeTests.cpp +++ b/cppForSwig/gtest/BridgeTests.cpp @@ -1832,7 +1832,7 @@ namespace { } bool connectToIp(std::shared_ptr bridge, - const std::string& ip, const std::string& port, + const std::string& ip, Network::port_t port, const std::string& expectedPubkey) { uint64_t refId = rand(); @@ -6803,7 +6803,7 @@ TEST_F(BridgeWalletsWithDBTests, Connect) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -6845,8 +6845,8 @@ TEST_F(BridgeWalletsWithDBTests, CycleConnection) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_FALSE(connectToIp(bridge_, "127.0.0.1", "9001", {})); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_FALSE(connectToIp(bridge_, "127.0.0.1", 9001, {})); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -6911,7 +6911,7 @@ TEST_F(BridgeWalletsWithDBTests, CycleConnection) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); theBDMt_->start(Config::DBSettings::initMode()); theBDMt_->bdm()->blockUntilReady(); @@ -6962,7 +6962,7 @@ TEST_F(BridgeWalletsWithDBTests, DeleteWallet) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7073,7 +7073,7 @@ TEST_F(BridgeWalletsWithDBTests, ExtendAddressChain) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7152,7 +7152,7 @@ TEST_F(BridgeWalletsWithDBTests, AddNewAddress) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7486,7 +7486,7 @@ TEST_F(BridgeChainDataTests, Check5Blocks_BCDE) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7531,7 +7531,7 @@ TEST_F(BridgeChainDataTests, ChangeFilters_ALFB_BCDE) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7614,7 +7614,7 @@ TEST_F(BridgeChainDataTests, BlocksOutOfOrder_BCDE) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7687,7 +7687,7 @@ TEST_F(BridgeChainDataTests, AddBlocks_BCDE) WebSocketServer::start(theBDMt_->bdm(), true); //connect to db - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7799,7 +7799,7 @@ TEST_F(BridgeChainDataTests, AddBlocks_BC_DE) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -7984,7 +7984,7 @@ TEST_F(BridgeChainDataTests, AddBlocks_BCDE_AFLB) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -8544,7 +8544,7 @@ TEST_F(BridgeChainDataTests, Reorg_BCDE) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -8686,7 +8686,7 @@ TEST_F(BridgeChainDataTests, Reorg_BCDE_DifferentOrder) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -8797,7 +8797,7 @@ TEST_F(BridgeChainDataTests, Reorg_BC_DE) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -9032,7 +9032,7 @@ TEST_F(BridgeChainDataTests, Reorg_BCDE_AFLB) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -9755,7 +9755,7 @@ TEST_F(BridgeChainDataTests, Reorg_SpendBeforeBranchPoint) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -10037,7 +10037,7 @@ TEST_F(BridgeChainDataTests, AddressBook) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -10117,7 +10117,7 @@ TEST_F(BridgeChainDataTests, getUTXOs) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -10249,7 +10249,7 @@ TEST_F(BridgeChainDataTests, ZeroConf) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -10483,7 +10483,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_Replace) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -10683,7 +10683,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_Chain) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -10908,7 +10908,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_StaggeredChain) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -11423,7 +11423,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_ChainRBF) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -11926,7 +11926,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_Reload) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -12116,7 +12116,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_Reload) }); loadWallets({walletId_BCDE_}); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); ASSERT_EQ(goOnline(bridge_), 5); @@ -12198,7 +12198,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_Reorg) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -12531,7 +12531,7 @@ TEST_F(BridgeChainDataTests, DISABLED_ZeroConf_RegisterWallet) //init bridge loadWallets({walletId_BCDE_}); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -12764,7 +12764,7 @@ TEST_F(BridgeChainDataTests, RestoreSynchronize) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -12967,7 +12967,7 @@ TEST_F(BridgeChainDataTests, RestoreSynchronize) } //go online - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); ASSERT_EQ(goOnline(bridge_), 5); @@ -13046,7 +13046,7 @@ TEST_F(BridgeChainDataTests, ZeroConf_SpendNew) homedir_ / SERVER_AUTH_PEER_FILENAME, authPeersPassLbd_}); WebSocketServer::start(theBDMt_->bdm(), true); - ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", "9001", serverPubkey_)); + ASSERT_TRUE(connectToIp(bridge_, "127.0.0.1", 9001, serverPubkey_)); ASSERT_TRUE(registerWallets(bridge_)); //start db, go online and wait on ready notif @@ -14253,7 +14253,7 @@ TEST_F(BridgePeersManagement, ListAddConnect) ASSERT_FALSE(connectToPeer(bridge_, serverKey)); //add the server to peers store - auto serverAddress = std::string{"127.0.0.1:"} + Config::NetworkSettings::dbPort(); + auto serverAddress = std::format("127.0.0.1:{}", Config::NetworkSettings::dbPort()); addPeer(serverPubkey_, { serverAddress }, "the server key"); //list again, server should appear @@ -14299,7 +14299,7 @@ TEST_F(BridgePeersManagement, LoadDeleteCreate) ASSERT_EQ(clientKey.size(), 48); //add the server to peers store - auto serverAddress = std::string{"127.0.0.1:"} + Config::NetworkSettings::dbPort(); + auto serverAddress = std::format("127.0.0.1:{}", Config::NetworkSettings::dbPort()); addPeer(serverPubkey_, { serverAddress }, "my serv key"); //list again, server should appear @@ -14345,7 +14345,7 @@ TEST_F(BridgePeersManagement, Remove) ASSERT_FALSE(connectToPeer(bridge_, "abcd")); //add the server to peers store - auto serverAddress = std::string{"127.0.0.1:"} + Config::NetworkSettings::dbPort(); + auto serverAddress = std::format("127.0.0.1:{}", Config::NetworkSettings::dbPort()); addPeer(serverPubkey_, { serverAddress }, "serv key"); //also add a random key diff --git a/qtdialogs/setupmanager/DatabaseTab.py b/qtdialogs/setupmanager/DatabaseTab.py index 67230c785..3335343a3 100644 --- a/qtdialogs/setupmanager/DatabaseTab.py +++ b/qtdialogs/setupmanager/DatabaseTab.py @@ -1050,7 +1050,7 @@ def collectSettings(self): peerKey = '' ipAddr = '' - ipPort = '' + dbPort = 0 if isPeer: selected = self.peerList.currentItem() @@ -1061,8 +1061,8 @@ def collectSettings(self): elif isIp: ipAddr = self.ipEdit.text().strip() portText = self.portEdit.text().strip() - ipPort = portText if portText \ - else str(ARMORYDB_DEFAULT_PORT) + dbPort = int(portText) if portText \ + else ARMORYDB_DEFAULT_PORT return { 'dbPath': self.databaseDirEdit.text(), @@ -1076,7 +1076,7 @@ def collectSettings(self): 'threads': self.threadCount, 'peerKey': peerKey, 'ipAddr': ipAddr, - 'ipPort': ipPort, + 'dbPort': dbPort } def validate(self): diff --git a/qtdialogs/setupmanager/DlgSetupManager.py b/qtdialogs/setupmanager/DlgSetupManager.py index f36eeec37..6f10b3b8c 100644 --- a/qtdialogs/setupmanager/DlgSetupManager.py +++ b/qtdialogs/setupmanager/DlgSetupManager.py @@ -470,9 +470,9 @@ def getDbConnectionParams(self): params['peerKey'] = dbSettings['peerKey'] elif scenario == SCENARIO_REMOTE_IP: params['ipAddr'] = dbSettings['ipAddr'] - params['ipPort'] = dbSettings['ipPort'] \ - if dbSettings['ipPort'] \ - else str(ARMORYDB_DEFAULT_PORT) + params['dbPort'] = dbSettings['dbPort'] \ + if dbSettings['dbPort'] \ + else ARMORYDB_DEFAULT_PORT return params @@ -568,17 +568,17 @@ def _connectToIp(self, params): 4. C++ finishes connecting (or fails) 5. Result callback fires with actual success/failure """ - ipAddr = params.get('ipAddr', '') - ipPort = params.get( - 'ipPort', str(ARMORYDB_DEFAULT_PORT)) + dbAddr = params.get('ipAddr', '') + dbPort = params.get( + 'dbPort', ARMORYDB_DEFAULT_PORT) - if not ipAddr: + if not dbAddr: raise ValueError( "ipAddr missing from params") - LOGINFO(f"Calling connectToIp: {ipAddr}:{ipPort}") + LOGINFO(f"Calling connectToIp: {dbAddr}:{dbPort}") - callbackId = f"connectToIp_{ipAddr}_{ipPort}" + callbackId = f"connectToIp_{dbAddr}_{dbPort}" self.pendingConnectionResult = None def onConnectResult(reply): @@ -595,7 +595,7 @@ def onConnectResult(reply): onPresentPubkey=self._onServerKeyPresented) TheBridge.dbSetup.connectToIp( - ip=ipAddr, port=ipPort, + ip=dbAddr, port=dbPort, callbackId=callbackId, resultCallback=onConnectResult) From a84578e41c9a763a8ea18a154cc138fb44b52cd9 Mon Sep 17 00:00:00 2001 From: goatpig Date: Mon, 21 Sep 2026 19:49:35 +0200 Subject: [PATCH 12/26] wstring path handling for WinAPI CreateProcessW --- cppForSwig/BridgeAPI/DBSetup.cpp | 75 +++++++++++++++++-- cppForSwig/BridgeAPI/DBSetup.h | 4 +- cppForSwig/BridgeAPI/Wallets/Manager.cpp | 2 +- .../BridgeAPI/Wallets/Notifications.cpp | 2 +- cppForSwig/BridgeAPI/Wallets/Notifications.h | 2 +- 5 files changed, 74 insertions(+), 11 deletions(-) diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index 47a48c56b..48db734d5 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -49,16 +49,80 @@ namespace { //figure out default windows install location }; + std::wstring toWString(const std::string& str) + { + auto wCharCount = MultiByteToWideChar( + CP_UTF8, 0, + str.c_str(), str.size(), + nullptr, 0 + ); + if (wCharCount == 0) { + throw std::runtime_error("could not project wchar size"); + } + + std::wstring wString; + wString.resize(wCharCount); + auto result = MultiByteToWideChar( + CP_UTF8, 0, + str.c_str(), str.size(), + wString.data(), wCharCount + ); + if (result != wCharCount) { + throw std::runtime_error("failed to convert to wstring"); + } + return wString; + } + + std::string getLastErrorVerbose() + { + auto lastError = GetLastError(); + LPVOID lpMsgBuf; + + if (FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR) &lpMsgBuf, + 0, NULL) == 0) { + return std::format("could not retrieve verbose for error code {}", lastError); + } + + std::string result{(LPCSTR)lpMsgBuf}; + LocalFree(lpMsgBuf); + return result; + } + + //////// std::pair spawnProcess( const std::filesystem::path& target, + const std::map& pathArgs, const std::vector& args, const std::map& envvars, bool captureStdOut) { - //use CreateProcess to spawn ArmoryDB + /* use CreateProcess to spawn ArmoryDB */ + + //binary target std::wstring commandLine{ target.wstring() }; - for (const auto& arg : args) { - commandLine.append(std::format("{} ", arg)); + + //path arguments + try { + for (const auto& pathArg : pathArgs) { + auto wArg = toWString(pathArg.first); + commandLine.append(std::format(L"{}={}", wArg, pathArg.second.wstring())); + } + + //other args + for (const auto& arg : args) { + auto wArg = toWString(arg); + commandLine.append(std::format(L"{} ", wArg)); + } + } catch (const std::exception& e) { + return { {}, std::format( + "failed to build arg string with error: {}", e.what()) }; } //mandatory, process handle is written inside pi after start @@ -93,8 +157,7 @@ namespace { NULL, &si, &pi )) { - auto lastError = GetLastError(); - return { {}, lastError }; + return { {}, getLastErrorVerbose() }; } } auto handle = pi.hProcess; @@ -654,7 +717,7 @@ AutomationContext::AutomationContext( automateNode_{automateNode}, automateDb_{automateDb} {} -uint32_t AutomationContext::getDbPort() const +Network::port_t AutomationContext::getDbPort() const { return dbPort_; } diff --git a/cppForSwig/BridgeAPI/DBSetup.h b/cppForSwig/BridgeAPI/DBSetup.h index b1c537d45..79e8f86f0 100644 --- a/cppForSwig/BridgeAPI/DBSetup.h +++ b/cppForSwig/BridgeAPI/DBSetup.h @@ -73,7 +73,7 @@ namespace Armory const bool automateDb_; std::shared_ptr peers_; - uint32_t dbPort_ = UINT32_MAX; + uint16_t dbPort_ = UINT16_MAX; bool hasRun_ = false; std::string rpcLogin_; @@ -101,7 +101,7 @@ namespace Armory bool isDbRunning(void); bool isSatoshiRunning(void); - uint32_t getDbPort(void) const; + uint16_t getDbPort(void) const; std::shared_ptr getPeerStore(void) const; }; diff --git a/cppForSwig/BridgeAPI/Wallets/Manager.cpp b/cppForSwig/BridgeAPI/Wallets/Manager.cpp index 2ae03c3fa..85f465918 100644 --- a/cppForSwig/BridgeAPI/Wallets/Manager.cpp +++ b/cppForSwig/BridgeAPI/Wallets/Manager.cpp @@ -249,7 +249,7 @@ void WalletManager::setBdvCallback( { switch (notif->type) { - case NotifType::REGISTERED: + case NotifType::IS_REGISTERED: { if (automatesDB_) { //if we automate the db, we have to tell it to start scanning diff --git a/cppForSwig/BridgeAPI/Wallets/Notifications.cpp b/cppForSwig/BridgeAPI/Wallets/Notifications.cpp index 6d270d4f4..e5acdb8c2 100644 --- a/cppForSwig/BridgeAPI/Wallets/Notifications.cpp +++ b/cppForSwig/BridgeAPI/Wallets/Notifications.cpp @@ -418,5 +418,5 @@ bool NotifStruct_NewBlock::syncWalletState() const //////// NotifStruct_Registered::NotifStruct_Registered() : - NotifStruct(NotifType::REGISTERED) + NotifStruct(NotifType::IS_REGISTERED) {} diff --git a/cppForSwig/BridgeAPI/Wallets/Notifications.h b/cppForSwig/BridgeAPI/Wallets/Notifications.h index a23a96141..9846241e4 100644 --- a/cppForSwig/BridgeAPI/Wallets/Notifications.h +++ b/cppForSwig/BridgeAPI/Wallets/Notifications.h @@ -31,7 +31,7 @@ namespace Armory NEWBLOCK, ZC, REFRESH, - REGISTERED, + IS_REGISTERED, DISCONNECTED }; From 093b78a884cf73db5f38e5919ec594d9b24e1ab9 Mon Sep 17 00:00:00 2001 From: goatpig Date: Wed, 23 Sep 2026 14:13:23 +0200 Subject: [PATCH 13/26] implement windows side stdout hijacking --- cppForSwig/BridgeAPI/DBSetup.cpp | 83 ++++++++++++++++++-------- cppForSwig/Utils/BtcUtils.cpp | 10 ++-- cppForSwig/Wallets/AuthorizedPeers.cpp | 1 - 3 files changed, 64 insertions(+), 30 deletions(-) diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index 48db734d5..567a0c294 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -112,13 +112,13 @@ namespace { try { for (const auto& pathArg : pathArgs) { auto wArg = toWString(pathArg.first); - commandLine.append(std::format(L"{}={}", wArg, pathArg.second.wstring())); + commandLine.append(std::format(L" {}={}", wArg, pathArg.second.wstring())); } //other args for (const auto& arg : args) { auto wArg = toWString(arg); - commandLine.append(std::format(L"{} ", wArg)); + commandLine.append(std::format(L" {}", wArg)); } } catch (const std::exception& e) { return { {}, std::format( @@ -144,25 +144,55 @@ namespace { SetEnvironmentVariable(envvar.first.c_str(), envvar.second.c_str()); } + HANDLE pipeRead = nullptr; + HANDLE pipeWrite = nullptr; if (captureStdOut) { - throw std::runtime_error("implement stdout capture in windows"); - } else { - if (!CreateProcessW(NULL, - commandLine.data(), - NULL, - NULL, - true, //inherit parent handles where possible - NORMAL_PRIORITY_CLASS, - NULL, //no explicit envvars on windows, let child inherit parent's - NULL, - &si, &pi - )) { - return { {}, getLastErrorVerbose() }; + //create pipes to take over stdout in child process + SECURITY_ATTRIBUTES sa; + ZeroMemory(&sa, sizeof(sa)); + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.bInheritHandle = true; + if (CreatePipe(&pipeRead, &pipeWrite, &sa, 0) == 0) { + return { {}, "failed to create pipes" }; } + + //do not let child inherit the read pipe + SetHandleInformation(pipeRead, HANDLE_FLAG_INHERIT, 0); + si.hStdOutput = pipeWrite; + si.hStrError = pipeWrite; + si.dwFlags |= STARTF_USESTDHANDLES; + } + if (CreateProcessW(NULL, + commandLine.data(), + NULL, + NULL, + true, //inherit parent handles where possible + NORMAL_PRIORITY_CLASS, + NULL, //no explicit envvars on windows, let child inherit parent's + NULL, + &si, &pi) == 0) { + return { {}, getLastErrorVerbose() }; } + + std::string stdOutStr; + if (captureStdOut) { + //parent side has no use for the write pipe + CloseHandle(pipeWrite); + + //grab first output to stdout and cleanup + stdOutStr.resize(2048); + DWORD bytesRead; + if (ReadFile(pipeRead, stdOutStr.data(), 2047, &bytesRead, nullptr) != 0) { + stdOutStr.resize(bytesRead); + } else { + stdOutStr.resize(0); + } + CloseHandle(pipeRead); + } + auto handle = pi.hProcess; CloseHandle(pi.hThread); - return { {handle}, {} }; + return { {handle}, stdOutStr }; } #else @@ -667,7 +697,7 @@ bool ProcessInstance::isRunning() } #ifdef _WIN32 - if (WaitForSingleObject(instance_) != WAIT_TIMEOUT) { + if (WaitForSingleObject(instance_, 0) != WAIT_TIMEOUT) { //we need to close this handle after use CloseHandle(instance_); instance_ = INVALID_INSTANCE; @@ -809,19 +839,19 @@ void AutomationContext::automateDb() LOGINFO << "spawning ArmoryDB"; /* - Spawn ArmoryDB with tailored CLI args and environment variables to setup - adhoc a AEAD 2-way handshake. + Spawn ArmoryDB with tailored CLI args and environment variables, to setup + an adhoc AEAD 2-way handshake. 2-way Keys are exchange via the following these steps: 1. CppBridge creates an ephemeral key store and adds its public key to - to ArmoryDB via . - 2. CppBridge spawn ArmoryDB, replacing stdout by a pipe. + to ArmoryDB via envvars. + 2. CppBridge spawns ArmoryDB, replacing stdout by a pipe. 3. ArmoryDB detects automation via the --ephemeral CLI arg. It creates an ephemeral key store, reads the caller pubkey from - envvars, adds it to the store and sets it as the store's master key. + envvars, adds it to its store and sets it as the store's master key. 4. ArmoryDB writes its public key to stdout. - 5. CppBridge detects changes to the key file, grabs the pubkey and - injects it into its own store. The pipe is cleaned up. + 5. CppBridge reads ArmoryDB's stdout, grabs the pubkey and + injects it into its own store. */ //sanity check @@ -830,8 +860,13 @@ void AutomationContext::automateDb() } //get full path to armorydb +#ifdef _WIN32 + const std::filesystem::path armoryDbPath{ + Config::Pathing::runningDir() / "ArmoryDB.exe" }; +#else const std::filesystem::path armoryDbPath{ Config::Pathing::runningDir() / "ArmoryDB" }; +#endif if (!FileUtils::pathExists(armoryDbPath, 8)) { throw std::runtime_error("invalid db binary path: " + armoryDbPath.string()); } diff --git a/cppForSwig/Utils/BtcUtils.cpp b/cppForSwig/Utils/BtcUtils.cpp index e8ad8aa2b..01974a91a 100644 --- a/cppForSwig/Utils/BtcUtils.cpp +++ b/cppForSwig/Utils/BtcUtils.cpp @@ -1442,7 +1442,7 @@ std::string BtcUtils::base64_decode(const std::string& in) size_t count = (in.size() + 3) / 4; std::string result; result.resize(count * 3); - auto ptr = in.c_str(); + const char* ptr = in.c_str(); auto result_ptr = (uint8_t*)result.c_str(); unsigned y=0; @@ -1458,14 +1458,14 @@ std::string BtcUtils::base64_decode(const std::string& in) ++i; } - auto val8 = ptr[y]; + const char val8 = ptr[y]; auto iter = base64Vals.find(val8); if (iter == base64Vals.end()) { - if (val8 == '=' || val8 == '\n') { + if (val8 == '=' || val8 == '\n' || val8 == 13) { break; } - throw std::runtime_error( - std::format("invalid b64 character: \"{}\"", val8)); + throw std::runtime_error(std::format( + "invalid b64 character: {}", (unsigned)val8)); } uint32_t bits = iter->second << (26 - (6 * (y % 4))); diff --git a/cppForSwig/Wallets/AuthorizedPeers.cpp b/cppForSwig/Wallets/AuthorizedPeers.cpp index 7f8eeb747..4da7ef8b4 100644 --- a/cppForSwig/Wallets/AuthorizedPeers.cpp +++ b/cppForSwig/Wallets/AuthorizedPeers.cpp @@ -803,7 +803,6 @@ PeerKey PeerKey::fromHumanReadable(const std::string& str) throw std::runtime_error("invalid peer key size"); } - auto header = brr.get_BinaryDataRef(3); uint32_t headerInt = uint32_t(header[0]) << 24 | From 7a57dd8fbe20c0cd6b12228a5703c2b33f79ae43 Mon Sep 17 00:00:00 2001 From: goatpig Date: Wed, 23 Sep 2026 14:13:43 +0200 Subject: [PATCH 14/26] fix BackupTests.BackupString_LegacyStatic on Windows --- cppForSwig/gtest/WalletTests.cpp | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cppForSwig/gtest/WalletTests.cpp b/cppForSwig/gtest/WalletTests.cpp index 69e14e282..6c4f05db2 100644 --- a/cppForSwig/gtest/WalletTests.cpp +++ b/cppForSwig/gtest/WalletTests.cpp @@ -11315,14 +11315,21 @@ TEST_F(BackupTests, BackupString_LegacyStatic) nullptr, 10 }); - auto woRestore = Seeds::Helpers::restoreFromBackup( - std::move(backupWO), callback, IO::CreateWalletParams{ - newHomeDir, - Passphrase::SetNew{}, - Passphrase::SetNew{1ms, 0, SecureBinaryData::fromString("woctrl")}, - nullptr, 10 - }); - auto filename = woRestore.wltPtr->getDbFilename(); + std::filesystem::path filename; + std::unique_ptr newBackupWO; + { + auto woRestore = Seeds::Helpers::restoreFromBackup( + std::move(backupWO), callback, IO::CreateWalletParams{ + newHomeDir, + Passphrase::SetNew{}, + Passphrase::SetNew{1ms, 0, SecureBinaryData::fromString("woctrl")}, + nullptr, 10 + }); + filename = woRestore.wltPtr->getDbFilename(); + auto woWltSingle = std::dynamic_pointer_cast( + woRestore.wltPtr); + newBackupWO = Seeds::Helpers::getWalletBackup(woWltSingle, false); + } auto wltSingle = std::dynamic_pointer_cast( fullRestore.wltPtr); @@ -11346,9 +11353,6 @@ TEST_F(BackupTests, BackupString_LegacyStatic) EXPECT_EQ(newFullBackE16->getChaincode(Seeds::LineIndex::Two, false), fullBackup[3]); EXPECT_TRUE(compareWalletWithBackup(wltSingle, filename, {}, "woctrl")); - auto woWltSingle = std::dynamic_pointer_cast( - woRestore.wltPtr); - auto newBackupWO = Seeds::Helpers::getWalletBackup(woWltSingle, false); auto newBackupPublic = dynamic_cast( newBackupWO.get()); ASSERT_NE(newBackupPublic, nullptr); From 368594f863c75c357b847ba7540f85d0864ef418 Mon Sep 17 00:00:00 2001 From: goatpig Date: Wed, 23 Sep 2026 15:21:41 +0200 Subject: [PATCH 15/26] fix Windows autodb startup sequence fix missing reply error on db start command pass BridgeTests build on Windows --- cppForSwig/BDM_Server.cpp | 2 +- cppForSwig/BridgeAPI/DBSetup.cpp | 2 +- cppForSwig/Network/WebSocketClient.cpp | 4 +-- cppForSwig/Server.cpp | 36 ++++++++++---------------- cppForSwig/gtest/BridgeTests.cpp | 2 +- 5 files changed, 19 insertions(+), 27 deletions(-) diff --git a/cppForSwig/BDM_Server.cpp b/cppForSwig/BDM_Server.cpp index 25a238968..8332f176e 100644 --- a/cppForSwig/BDM_Server.cpp +++ b/cppForSwig/BDM_Server.cpp @@ -382,7 +382,7 @@ namespace { break; } clients->bdm()->signalStart(true); - break; + return nullptr; } case StaticRequest::Which::SHUTDOWN: diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index 567a0c294..2f8685267 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -159,7 +159,7 @@ namespace { //do not let child inherit the read pipe SetHandleInformation(pipeRead, HANDLE_FLAG_INHERIT, 0); si.hStdOutput = pipeWrite; - si.hStrError = pipeWrite; + si.hStdError = pipeWrite; si.dwFlags |= STARTF_USESTDHANDLES; } if (CreateProcessW(NULL, diff --git a/cppForSwig/Network/WebSocketClient.cpp b/cppForSwig/Network/WebSocketClient.cpp index 6453e2e6f..9b7c1a92e 100755 --- a/cppForSwig/Network/WebSocketClient.cpp +++ b/cppForSwig/Network/WebSocketClient.cpp @@ -106,7 +106,7 @@ void WebSocketClient::pushPayload( //set response id readPackets_.insert(make_pair(write_payload->id, move(response))); } - writeSerializationQueue_.push_back(move(write_payload)); + writeSerializationQueue_.push_back(std::move(write_payload)); } void WebSocketClient::writeService() @@ -521,7 +521,7 @@ void WebSocketClient::readService() readPackets_.erase(msgid); currentReadMessage_.reset(); } else { - LOGWARN << "invalid msg id: " << msgid; + LOGWARN << "there is no callback registered for msg id: " << msgid; currentReadMessage_.reset(); } } diff --git a/cppForSwig/Server.cpp b/cppForSwig/Server.cpp index 7fd81f3c2..61008c62a 100755 --- a/cppForSwig/Server.cpp +++ b/cppForSwig/Server.cpp @@ -233,11 +233,9 @@ void WebSocketServer::initPeerStore( auto masterKey = NetworkPeers::PeerKey::fromHumanReadable({keyPtr}); //inject master pubkey in the store - instance->peerStore_->addPeer( - masterKey, - {std::format("127.0.0.1:{}", Config::NetworkSettings::dbPort())}, - {} - ); + instance->peerStore_->addPeer(masterKey, { + std::format("127.0.0.1:{}", Config::NetworkSettings::dbPort()) + }, {}); //set master key if (!instance->peerStore_->setMasterKey(masterKey)) { @@ -389,6 +387,17 @@ void WebSocketServer::setIsReady() /////////////////////////////////////////////////////////////////////////////// void WebSocketServer::webSocketService(int port) { + if (Config::NetworkSettings::ephemeralPeers()) { + NetworkPeers::PeerKey myKey{ + peerStore_->getOwnPublicKey(), + NetworkPeers::PeerType::ServerTwoWay + }; + std::cout << myKey.toHumanReadable() << std::endl; + + //set stdout to nullptr to suppress any further cout + std::cout.rdbuf(nullptr); + } + struct lws_context_creation_info info; struct lws_vhost *vhost; const char *iface = nullptr; @@ -424,23 +433,6 @@ void WebSocketServer::webSocketService(int port) pendingWritesIter_ = pendingWrites_.begin(); run_.store(1, std::memory_order_relaxed); - if (Config::NetworkSettings::ephemeralPeers()) { - /* - DB is automated by client, output pubkey to stdout to complete AEAD - key share. - We do this at this stage to make sure the server is ready and listening - before the client tries to connect. - */ - NetworkPeers::PeerKey myKey{ - peerStore_->getOwnPublicKey(), - NetworkPeers::PeerType::ServerTwoWay - }; - std::cout << myKey.toHumanReadable() << std::endl; - - //set stdout to nullptr to suppress any further cout - std::cout.rdbuf(nullptr); - } - try { while (run_.load(std::memory_order_relaxed) != 0 && n >= 0) { n = lws_service(contextPtr_, 10000); diff --git a/cppForSwig/gtest/BridgeTests.cpp b/cppForSwig/gtest/BridgeTests.cpp index e8273ada7..f153fb8a1 100644 --- a/cppForSwig/gtest/BridgeTests.cpp +++ b/cppForSwig/gtest/BridgeTests.cpp @@ -13583,7 +13583,7 @@ class BridgeBlocksAutoDBTests : public ::testing::Test } auto validationReply = helperReply.getValidateDir(); - return std::filesystem::path{validationReply.getPath()}; + return std::filesystem::path{std::string{validationReply.getPath()}}; } protected: From 2256ef6420ff2fdeb74f6fdea45bd82e9a421239 Mon Sep 17 00:00:00 2001 From: goatpig Date: Thu, 24 Sep 2026 11:12:05 +0200 Subject: [PATCH 16/26] give armorydb time to start in automate operations --- cppForSwig/BridgeAPI/DBSetup.cpp | 38 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index 2f8685267..0dd584231 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -527,25 +527,33 @@ BdvPtr Armory::Bridge::setupClientConnection( } //setup bdv obj - BdvPtr bdvPtr = AsyncClient::BlockDataViewer::getNewBDV( - ip, port, - peers, oneWayAuth, - cbPtr - ); + BdvPtr bdvPtr; + unsigned count = 0; + while (count++ < 10) { + bdvPtr = AsyncClient::BlockDataViewer::getNewBDV( + ip, port, + peers, oneWayAuth, + cbPtr + ); - if (presentPubKeyFunc) { - bdvPtr->setCheckServerKeyPromptLambda(presentPubKeyFunc); - } + if (presentPubKeyFunc) { + bdvPtr->setCheckServerKeyPromptLambda(presentPubKeyFunc); + } + + //connect to db + if (!bdvPtr->connectToRemote()) { + //could not connect, sleep for 250ms and try again + std::this_thread::sleep_for(250ms); + } + bdvPtr->registerWithDB( + Config::BitcoinSettings::getMagicBytes().toHexStr()); - //connect to db - if (!bdvPtr->connectToRemote()) { - return nullptr; + //notify setup is done + return bdvPtr; } - bdvPtr->registerWithDB( - Config::BitcoinSettings::getMagicBytes().toHexStr()); - //notify setup is done - return bdvPtr; + LOGERR << "failed to connect to armorydb"; + return nullptr; } BdvPtr Armory::Bridge::setupClientConnection( From 0dbfca1714512ae4c8266c2f030d906f31a1b2d3 Mon Sep 17 00:00:00 2001 From: goatpig Date: Fri, 25 Sep 2026 19:57:21 +0200 Subject: [PATCH 17/26] fix socket write queue in Windows get rid of deprecated ArmoryQt shutdown sequence append .exe suffix to bridge binary in windows --- ArmoryQt.py | 1 - armoryengine/BDM.py | 17 ----- armoryengine/CppBridge.py | 19 +++--- cppForSwig/Network/SocketObject.cpp | 102 +++++++++++++--------------- 4 files changed, 56 insertions(+), 83 deletions(-) diff --git a/ArmoryQt.py b/ArmoryQt.py index 759561ef2..4ce8592e6 100755 --- a/ArmoryQt.py +++ b/ArmoryQt.py @@ -4797,7 +4797,6 @@ def completeCloseForReal(self): else: LOGINFO('BDM is safe for clean shutdown') - TheBDM.shutdown() # Remove Temp Modules Directory if it exists: if self.tempModulesDirName: shutil.rmtree(self.tempModulesDirName) diff --git a/armoryengine/BDM.py b/armoryengine/BDM.py index ce907da86..9924ec9bd 100644 --- a/armoryengine/BDM.py +++ b/armoryengine/BDM.py @@ -181,23 +181,6 @@ def setState(self, state): def getState(self): return self.bdmState - ############################################################################# - @ActLikeASingletonBDM - def shutdown(self): - if self.bdmState == BDM_OFFLINE: - return - - try: - if CLI_OPTIONS.bip150Used or CLI_OPTIONS.bip151Used: - Cpp.DisableBIP151() - self.bdv_.unregisterFromDB() - self.callback.shutdown() - - cookie = self.getCookie() - self.bdv_.shutdown(cookie) - except: - pass - ############################################################################# @ActLikeASingletonBDM def RegisterEventForSignal(self, func, signal): diff --git a/armoryengine/CppBridge.py b/armoryengine/CppBridge.py index c188acd33..6a0660681 100755 --- a/armoryengine/CppBridge.py +++ b/armoryengine/CppBridge.py @@ -1,17 +1,16 @@ ################################################################################ # # -# Copyright (C) 2019-2024, goatpig. # +# Copyright (C) 2019-2026, goatpig. # # Distributed under the MIT license # # See LICENSE-MIT or https://opensource.org/licenses/MIT # # # ################################################################################ -from __future__ import (absolute_import, division, annotations, - print_function, unicode_literals) import os import errno import socket -from armoryengine.ArmoryUtils import LOGDEBUG, LOGERROR, LOGWARN, hash256, LOGINFO +from armoryengine.ArmoryUtils import LOGDEBUG, LOGERROR, LOGWARN, LOGINFO, \ + OS_WINDOWS from armoryengine.BinaryPacker import BinaryPacker, \ UINT32, UINT8, BINARY_CHUNK, VAR_INT from struct import unpack @@ -69,11 +68,12 @@ class BridgeSignerError(Exception): ################################################################################ def findCppBridgeBinary() -> str: #search candidate locations in priority order, return the first that exists + bridgeBinName = "CppBridge.exe" if OS_WINDOWS else "CppBridge" candidates = [ os.path.normpath(os.path.join( - os.path.dirname(os.path.abspath(__file__)), '..', 'CppBridge')), - os.path.join(os.getcwd(), 'CppBridge'), - os.path.join(os.getcwd(), 'build', 'CppBridge'), + os.path.dirname(os.path.abspath(__file__)), '..', bridgeBinName)), + os.path.join(os.getcwd(), bridgeBinName), + os.path.join(os.getcwd(), 'build', bridgeBinName), ] for candidate in candidates: @@ -228,9 +228,8 @@ def sendToBridgeBinary(self, payload, payloadId, needsReply=True, callback: callable=None, cbArgs: list=[], msgType = BRIDGE_CLIENT_HEADER): - #grab id from msg counter if self.run == False: - return + return None #serialize payload bp = BinaryPacker() @@ -1619,7 +1618,7 @@ def stop(self): ############################################################################# def send(self, msg, needsReply=True, callback=None, cbArgs=[], msgType=BRIDGE_CLIENT_HEADER): - self.bridgeSocket.sendToBridgeProto(msg, + return self.bridgeSocket.sendToBridgeProto(msg, needsReply, callback, cbArgs, msgType) ############################################################################# diff --git a/cppForSwig/Network/SocketObject.cpp b/cppForSwig/Network/SocketObject.cpp index 7921932ad..d8d1742fa 100644 --- a/cppForSwig/Network/SocketObject.cpp +++ b/cppForSwig/Network/SocketObject.cpp @@ -458,60 +458,48 @@ void PersistentSocket::socketService_win() size_t readIncrement = 8192; DWORD timeout = 100000; - WSAEventSelect(sockfd_, events_[0], FD_READ | FD_WRITE | FD_CLOSE); - bool writeReady = false; - - auto serviceSocketWrite = [&writeReady, this](void)->void + auto serviceSocketWrite = [this]() { - if (!writeReady) { - return; - } - std::vector payload; - if (!writeLeftOver_.empty()) { - payload = std::move(writeLeftOver_); - writeLeftOver_.clear(); - } else { - try { - payload = std::move(writeQueue_.pop_front()); - } catch (const Threading::IsEmpty&) { - return; + while (writeQueue_.count() > 0) { + std::vector payload; + if (!writeLeftOver_.empty()) { + payload = std::move(writeLeftOver_); + writeLeftOver_.clear(); + } else { + try { + payload = std::move(writeQueue_.pop_front()); + writeOffset_ = 0; + } catch (const Threading::IsEmpty&) { + return; + } } - } - WSABUF wsaBuffer; - wsaBuffer.buf = (char*)&payload[0] + writeOffset_; - wsaBuffer.len = payload.size() - writeOffset_; + WSABUF wsaBuffer; + wsaBuffer.buf = (char*)&payload[0] + writeOffset_; + wsaBuffer.len = payload.size() - writeOffset_; - DWORD bytessent; - if (WSASend(sockfd_, &wsaBuffer, 1, &bytessent, 0, nullptr, nullptr) == - SOCKET_ERROR) { - auto wsaError = WSAGetLastError(); - if (wsaError == WSAEWOULDBLOCK) { - writeReady = false; - } else { - LOGERR << "WSASend error with code: " << wsaError; - writeOffset_ = 0; - writeLeftOver_.clear(); + DWORD bytessent; + if (WSASend(sockfd_, &wsaBuffer, 1, &bytessent, 0, nullptr, nullptr) == + SOCKET_ERROR) { + writeLeftOver_ = std::move(payload); return; } - } else { - if (bytessent == 0) { - LOGWARN << "failed to write to socket, aborting"; - } - } - writeOffset_ += bytessent; - if (writeOffset_ < payload.size()) { - writeLeftOver_ = std::move(payload); - } else { - writeOffset_ = 0; + writeOffset_ += bytessent; + if (writeOffset_ < payload.size()) { + writeLeftOver_ = std::move(payload); + if (bytessent == 0) { + return; + } + } else { + writeOffset_ = 0; + } } }; - bool loop = true; - while (loop) { - serviceSocketWrite(); - auto ev = WSAWaitForMultipleEvents(1, events_, false, timeout, false); + WSAEventSelect(sockfd_, events_[0], FD_READ | FD_WRITE | FD_CLOSE); + while (true) { + auto ev = WSAWaitForMultipleEvents(2, events_, false, timeout, false); if (ev == WSA_WAIT_TIMEOUT) { continue; } @@ -521,21 +509,26 @@ void PersistentSocket::socketService_win() break; } - if (ev == WSA_WAIT_EVENT_0) { - //reset user event - WSAResetEvent(events_[0]); + int index; + if (ev >= WSA_WAIT_EVENT_0) { + index = ev - WSA_WAIT_EVENT_0; + } + + if (index == 1) + WSAResetEvent(events_[1]); + serviceSocketWrite(); + continue; } WSANETWORKEVENTS networkevents; - if (WSAEnumNetworkEvents(sockfd_, 0, - &networkevents) == SOCKET_ERROR) { + if (WSAEnumNetworkEvents(sockfd_, events_[0], &networkevents) == + SOCKET_ERROR) { LOGERR << "error getting network events for socket"; break; } //service socket - if (networkevents.lNetworkEvents & FD_READ) - { + if (networkevents.lNetworkEvents & FD_READ) { //read socket std::vector readdata; readdata.resize(readIncrement); @@ -551,7 +544,6 @@ void PersistentSocket::socketService_win() break; } LOGERR << "error reading socket, aborting"; - loop = false; break; } @@ -569,7 +561,7 @@ void PersistentSocket::socketService_win() } if (networkevents.lNetworkEvents & FD_WRITE) { - writeReady = true; + serviceSocketWrite(); } if (networkevents.lNetworkEvents & FD_CLOSE) { @@ -631,7 +623,7 @@ void PersistentSocket::signalService(uint8_t signal) if (signal == 1) { run_.store(false, std::memory_order_relaxed); } - WSASetEvent(events_[0]); + WSASetEvent(events_[1]); #else if (pipes_[1] == SOCK_MAX) { return; @@ -658,7 +650,7 @@ void PersistentSocket::initPipes() cleanUpPipes(); #ifdef _WIN32 - for (unsigned i = 0; i < 1; i++) { + for (unsigned i = 0; i < 2; i++) { events_[i] = WSACreateEvent(); } #else From f765f6a16e9fb12d585fe0b44f88f3197577a49a Mon Sep 17 00:00:00 2001 From: goatpig Date: Fri, 25 Sep 2026 20:14:09 +0200 Subject: [PATCH 18/26] fix missing accolade --- cppForSwig/Network/SocketObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppForSwig/Network/SocketObject.cpp b/cppForSwig/Network/SocketObject.cpp index d8d1742fa..0951ac9ff 100644 --- a/cppForSwig/Network/SocketObject.cpp +++ b/cppForSwig/Network/SocketObject.cpp @@ -514,7 +514,7 @@ void PersistentSocket::socketService_win() index = ev - WSA_WAIT_EVENT_0; } - if (index == 1) + if (index == 1) { WSAResetEvent(events_[1]); serviceSocketWrite(); continue; From 9ae700c5937fc65fe842193a48d7a392ffa49de8 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sat, 26 Sep 2026 15:36:05 +0200 Subject: [PATCH 19/26] improve test coverage around zeroconf loading --- cppForSwig/Utils/Types.cpp | 4 +- cppForSwig/gtest/UtilsTests.cpp | 1 + cppForSwig/gtest/ZeroConfTests.cpp | 155 ++++++++++++++++++++++++++++- 3 files changed, 157 insertions(+), 3 deletions(-) diff --git a/cppForSwig/Utils/Types.cpp b/cppForSwig/Utils/Types.cpp index 98d0b0689..667113c4a 100644 --- a/cppForSwig/Utils/Types.cpp +++ b/cppForSwig/Utils/Types.cpp @@ -81,7 +81,9 @@ TxKey Types::getTxKeyFromTxIOKey(TxIOKey key) //////// bool Types::isThisAZCKey(TxKey key) { - return (key & 0x000000000000FFFF) == 0x000000000000FFFF; + return key != INVALID_TX_KEY ? + (key & 0x000000000000FFFF) == 0x000000000000FFFF : + false; } bool Types::isThisATxIOKey(TxIOKey key) diff --git a/cppForSwig/gtest/UtilsTests.cpp b/cppForSwig/gtest/UtilsTests.cpp index cd3762317..bb4f67ace 100644 --- a/cppForSwig/gtest/UtilsTests.cpp +++ b/cppForSwig/gtest/UtilsTests.cpp @@ -4679,6 +4679,7 @@ TEST_F(TypesTests, keys) { auto invalidTxKey = Types::INVALID_TX_KEY; ASSERT_FALSE(Types::isTxKeyValid(invalidTxKey)); + ASSERT_FALSE(Types::isThisAZCKey(invalidTxKey)); auto invalidTxIOKey = Types::INVALID_TXIO_KEY; ASSERT_FALSE(Types::isTxIOKeyValid(invalidTxIOKey)); diff --git a/cppForSwig/gtest/ZeroConfTests.cpp b/cppForSwig/gtest/ZeroConfTests.cpp index 3f7819ce1..4bc5ebbb6 100644 --- a/cppForSwig/gtest/ZeroConfTests.cpp +++ b/cppForSwig/gtest/ZeroConfTests.cpp @@ -1569,7 +1569,92 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); - // + //check the two ZCs are in the mempool + { + auto zcParser = bdm->zeroConfCont(); + auto ss = zcParser->getSnapshot(); + + auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); + auto zcKey0 = ss->getKeyForHash(zcHash0); + ASSERT_TRUE(Types::isTxKeyValid(zcKey0)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey0)); + auto zcId0 = Types::getZcIdFromTxKey(zcKey0); + ASSERT_EQ(zcId0, 0); + + auto zcHash1 = rawLBZcVec.zcVec_[0].first.getThisHash(); + auto zcKey1 = ss->getKeyForHash(zcHash1); + ASSERT_TRUE(Types::isTxKeyValid(zcKey1)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey1)); + auto zcId1 = Types::getZcIdFromTxKey(zcKey1); + ASSERT_EQ(zcId1, 1); + + ASSERT_EQ(ss->getTopZcID(), 1); + } + + //restart BDM + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + initBDM(); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 3U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash3); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash3)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 65 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //check the two ZCs are in the mempool + { + auto zcParser = bdm->zeroConfCont(); + auto ss = zcParser->getSnapshot(); + + auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); + auto zcKey0 = ss->getKeyForHash(zcHash0); + ASSERT_TRUE(Types::isTxKeyValid(zcKey0)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey0)); + auto zcId0 = Types::getZcIdFromTxKey(zcKey0); + ASSERT_EQ(zcId0, 0); + + auto zcHash1 = rawLBZcVec.zcVec_[0].first.getThisHash(); + auto zcKey1 = ss->getKeyForHash(zcHash1); + ASSERT_TRUE(Types::isTxKeyValid(zcKey1)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey1)); + auto zcId1 = Types::getZcIdFromTxKey(zcKey1); + ASSERT_EQ(zcId1, 1); + + ASSERT_EQ(ss->getTopZcID(), 1); + } + + //add last 2 blocks TestUtils::setBlocks({ "0", "1", "2", "3", "4", "5" }, blk0dat_); DBTestUtils::triggerNewBlockNotification(theBDMt_); DBTestUtils::waitOnNewBlockSignal(clients_, bdvID); @@ -1586,6 +1671,72 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 30 * COIN); EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 0 * COIN); + //check the two ZCs are out of the mempool + { + auto zcParser = bdm->zeroConfCont(); + auto ss = zcParser->getSnapshot(); + + auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); + auto zcKey0 = ss->getKeyForHash(zcHash0); + ASSERT_FALSE(Types::isTxKeyValid(zcKey0)); + ASSERT_FALSE(Types::isThisAZCKey(zcKey0)); + + auto zcHash1 = rawLBZcVec.zcVec_[0].first.getThisHash(); + auto zcKey1 = ss->getKeyForHash(zcHash1); + ASSERT_FALSE(Types::isTxKeyValid(zcKey1)); + ASSERT_FALSE(Types::isThisAZCKey(zcKey1)); + + ASSERT_EQ(ss->getTopZcID(), 1); + } + + //restart BDM again + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + initBDM(); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 5U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash5); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash5)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 25 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 0 * COIN); + + //check mempool is empty + { + auto zcParser = bdm->zeroConfCont(); + auto ss = zcParser->getSnapshot(); + ASSERT_EQ(ss, nullptr); + } + //cleanup bdvPtr.reset(); } @@ -9188,7 +9339,7 @@ TEST_F(ZeroConfTests_Supernode_WebSocket, BatchZcChain_ConflictingChildren_Alrea pCallback->waitOnZc(theBDMt_->bdm()->zeroConfCont(), txHashes); } - //batch push first zc (already in chain), C (unrelated) + //batch push first zc (already in chain), C (unrelated) //and tx3 (child of first, mempool conflict with tx2) bdvObj->broadcastZC({ rawTx1_B, rawTx1_C, rawTx3 }); std::set txHashes {tx1_C.getThisHash()}; From 806765f9838c4a10b51f21e23d18ada126fb36b2 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sun, 27 Sep 2026 14:19:51 +0200 Subject: [PATCH 20/26] correctly evict mined zc during loadMempool, fix clear-mempool --- cppForSwig/ZeroConf/Parser.cpp | 117 ++++--- cppForSwig/ZeroConf/Parser.h | 10 +- cppForSwig/ZeroConf/Utils.cpp | 59 ++-- cppForSwig/ZeroConf/Utils.h | 2 + cppForSwig/gtest/ZeroConfTests.cpp | 490 ++++++++++++++++++++++++++++- 5 files changed, 595 insertions(+), 83 deletions(-) diff --git a/cppForSwig/ZeroConf/Parser.cpp b/cppForSwig/ZeroConf/Parser.cpp index 32e910d58..0afe3f81f 100644 --- a/cppForSwig/ZeroConf/Parser.cpp +++ b/cppForSwig/ZeroConf/Parser.cpp @@ -558,11 +558,13 @@ void ZeroConfContainer::parseNewZC( std::unique_lock lock(parserMutex_); ZcUpdateBatch batch; + std::set droppedZcKeys; auto iter = zcMap.begin(); while (iter != zcMap.end()) { if (iter->second->state == ParsedTxStatus::Mined || iter->second->state == ParsedTxStatus::Invalid || iter->second->state == ParsedTxStatus::Skip) { + droppedZcKeys.emplace(iter->first); zcMap.erase(iter++); } else { ++iter; @@ -595,7 +597,6 @@ void ZeroConfContainer::parseNewZC( //zc logic std::set addedZcKeys; - std::set droppedZcKeys; for (const auto& newZCPair : zcMap) { const auto& txHash = newZCPair.second->getTxHash(); if (Types::isTxKeyValid(ss->getKeyForHash(txHash))) { @@ -876,61 +877,65 @@ void ZeroConfContainer::updateZCinDB() } catch (const Threading::StopBlockingLoop&) { break; } + updateZCinDB(std::move(batch)); + } +} - if (!batch.hasData()) { - continue; - } +void ZeroConfContainer::updateZCinDB(ZcUpdateBatch batch) +{ + if (!batch.hasData()) { + return; + } + auto tx = db_->beginTransaction(DB_SELECT::ZERO_CONF, LMDB::Mode::ReadWrite); - auto tx = db_->beginTransaction( - DB_SELECT::ZERO_CONF, LMDB::Mode::ReadWrite); - for (auto& zcPair : batch.zcToWrite) { - /*TODO: speed this up*/ - StoredTx zcTx; - auto txObj = zcPair.second->getTxObj(); - zcTx.createFromTx(txObj, true, true); - db_->putStoredZC(zcTx, zcPair.first); - } + for (auto& zcPair : batch.zcToWrite) { + /*TODO: speed this up*/ + StoredTx zcTx; + auto txObj = zcPair.second->getTxObj(); + zcTx.createFromTx(txObj, true, true); + db_->putStoredZC(zcTx, zcPair.first); + } - for (const auto& txhash : batch.txHashes) { - //if the key is not to be found in the txMap_, this is a ZC txhash - tx->insert( - LMDB::DataRef{txhash.getSize(), txhash.getPtr()}, - LMDB::DataRef{0, (const char*)nullptr} - ); + for (const auto& txhash : batch.txHashes) { + //if the key is not to be found in the txMap_, this is a ZC txhash + tx->insert( + LMDB::DataRef{txhash.getSize(), txhash.getPtr()}, + LMDB::DataRef{0, (const char*)nullptr} + ); + } + + for (auto& key : batch.keysToDelete) { + BinaryData keyWithPrefix; + keyWithPrefix.resize(7); + keyWithPrefix[0] = (uint8_t)DbPrefix::ZCDATA; + std::memcpy(keyWithPrefix.getPtr() + 1, &key, 6); + auto dbIter = tx->getIterator(); + if (!dbIter.seekToStartsWith(keyWithPrefix.getRef())) { + continue; } - for (auto& key : batch.keysToDelete) { - BinaryData keyWithPrefix; - keyWithPrefix.resize(7); - keyWithPrefix[0] = (uint8_t)DbPrefix::ZCDATA; - std::memcpy(keyWithPrefix.getPtr() + 1, &key, 6); - auto dbIter = tx->getIterator(); - if (!dbIter.seekToStartsWith(keyWithPrefix.getRef())) { - continue; + std::set ktd; + do { + auto thisKey = dbIter.getKeyRef(); + if (!thisKey.startsWith(keyWithPrefix.getRef())) { + break; } + ktd.emplace(thisKey); + } while (dbIter.advanceAndRead(DbPrefix::ZCDATA)); - std::set ktd; - do { - auto thisKey = dbIter.getKeyRef(); - if (!thisKey.startsWith(keyWithPrefix.getRef())) { - break; - } - ktd.emplace(thisKey); - } while (dbIter.advanceAndRead(DbPrefix::ZCDATA)); - - for (const auto& _key : ktd) { - tx->erase(LMDB::DataRef{_key.getSize(), _key.getPtr()}); - } + for (const auto& _key : ktd) { + tx->erase(LMDB::DataRef{_key.getSize(), _key.getPtr()}); } + } - for (const auto& _hash : batch.txHashesToDelete) { - tx->erase(LMDB::DataRef{_hash.getSize(), _hash.getPtr()}); - } - batch.setCompleted(true); + for (const auto& _hash : batch.txHashesToDelete) { + tx->erase(LMDB::DataRef{_hash.getSize(), _hash.getPtr()}); } + batch.setCompleted(true); } -unsigned ZeroConfContainer::loadZeroConfMempool(bool clearMempool) +//////// +unsigned ZeroConfContainer::loadMempool(bool clearMempool) { unsigned topId = 0; std::map> zcMap; @@ -986,13 +991,10 @@ unsigned ZeroConfContainer::loadZeroConfMempool(bool clearMempool) if (clearMempool == true) { LOGWARN << "Mempool was flagged for deletion!"; ZcUpdateBatch batch; - auto fut = batch.getCompletedFuture(); - for (const auto& zcTx : zcMap) { batch.keysToDelete.emplace(zcTx.first); } - updateBatch_.push_back(std::move(batch)); - fut.wait(); + updateZCinDB(std::move(batch)); } else if (!zcMap.empty()) { LOGDEBUG << "parsing " << zcMap.size() << " txns from mempool"; preprocessZcMap(zcMap, db_, bd_); @@ -1020,7 +1022,7 @@ void ZeroConfContainer::init(std::shared_ptr saf, LOGINFO << "Enabling zero-conf tracking"; scrAddrMap_ = saf->getZcFilterMapPtr(); - auto topId = loadZeroConfMempool(clearMempool); + auto topId = loadMempool(clearMempool); actionQueue_ = std::make_unique( [this](ZcActionStruct zas){ parseNewZC(std::move(zas)); }, zcPreprocessQueue_, topId); @@ -1673,6 +1675,9 @@ BatchTxMap ZeroConfContainer::getBatchTxMap( unsigned ZeroConfContainer::getMatcherMapSize() const { + if (actionQueue_ == nullptr) { + return UINT32_MAX; + } return actionQueue_->getMatcherMapSize(); } @@ -1685,6 +1690,14 @@ unsigned ZeroConfContainer::getMergeCount() const return ss->getMergeCount(); } +uint32_t ZeroConfContainer::getTopZcID() const +{ + if (actionQueue_ == nullptr) { + return UINT32_MAX; + } + return actionQueue_->getTopZcID(); +} + //////////////////////////////////////////////////////////////////////////////// // ZcActionQueue ZcActionQueue::ZcActionQueue( @@ -1920,11 +1933,17 @@ void ZcActionQueue::getDataToBatchMatcherThread() } } +//////// unsigned ZcActionQueue::getMatcherMapSize() const { return matcherMapSize_.load(std::memory_order_relaxed); } +uint32_t ZcActionQueue::getTopZcID() const +{ + return topId_.load(std::memory_order_relaxed); +} + //////////////////////////////////////////////////////////////////////////////// // ZcUpdateBatch std::shared_future ZcUpdateBatch::getCompletedFuture() diff --git a/cppForSwig/ZeroConf/Parser.h b/cppForSwig/ZeroConf/Parser.h index c35732933..d3ac42d5f 100644 --- a/cppForSwig/ZeroConf/Parser.h +++ b/cppForSwig/ZeroConf/Parser.h @@ -300,7 +300,9 @@ namespace Armory void queueGetDataResponse(std::shared_ptr); void queueBatch(std::shared_ptr); + //for unit tests unsigned getMatcherMapSize(void) const; + uint32_t getTopZcID(void) const; }; //////// @@ -355,7 +357,7 @@ namespace Armory std::shared_ptr) const; void increaseParserThreadPool(unsigned); - unsigned loadZeroConfMempool(bool); + unsigned loadMempool(bool); void reset(void); std::map> purge( @@ -394,7 +396,8 @@ namespace Armory std::shared_ptr); void updateZCinDB(void); - void handleInvTx(); + void updateZCinDB(ZcUpdateBatch); + void handleInvTx(void); BatchTxMap getBatchTxMap( std::shared_ptr, @@ -408,7 +411,6 @@ namespace Armory //action queue std::shared_future> pushNewBlockNotification(ReorganizationState); - unsigned getMatcherMapSize(void) const; // setup methods void init(std::shared_ptr, bool); @@ -449,6 +451,8 @@ namespace Armory //for unit tests unsigned getMergeCount(void) const; + unsigned getMatcherMapSize(void) const; + uint32_t getTopZcID(void) const; }; } //namespace ZeroConf } //namespace Armory diff --git a/cppForSwig/ZeroConf/Utils.cpp b/cppForSwig/ZeroConf/Utils.cpp index bc93626ad..a170ccd68 100644 --- a/cppForSwig/ZeroConf/Utils.cpp +++ b/cppForSwig/ZeroConf/Utils.cpp @@ -693,6 +693,20 @@ std::shared_ptr MempoolData::getTx(Types::TxKey key) const return iter->second; } +size_t MempoolData::getTxCount() const +{ + size_t count = 0; + for (const auto& txPair : txMap_) { + if (txPair.second != nullptr) { + ++count; + } + } + if (parent_ != nullptr) { + count += parent_->getTxCount(); + } + return count; +} + /////////////////////////////////////////////////////////////////////////////// Types::TxKey MempoolData::getKeyForHash(const Types::TxHash& hash) const { @@ -1068,7 +1082,17 @@ void MempoolSnapshot::preprocessZcMap( ZeroConf::preprocessZcMap(data_->txMap_, db, bd); } -/////////////////////////////////////////////////////////////////////////////// +Types::ZcId MempoolSnapshot::getTopZcID() const +{ + return topID_; +} + +size_t MempoolSnapshot::getTxCount() const +{ + return data_->getTxCount(); +} + +//////// std::shared_ptr MempoolSnapshot::getTxByKey_NoConst( Types::TxKey key) const { @@ -1082,16 +1106,17 @@ std::shared_ptr MempoolSnapshot::getTxByKey( return std::const_pointer_cast(txPtr); } -std::shared_ptr MempoolSnapshot::getTxByHash( - const Types::TxHash& hash) const +const Types::TxHash& MempoolSnapshot::getHashForKey( + Types::TxKey key) const { - auto key = getKeyForHash(hash); - if (!Types::isTxKeyValid(key)) { - return nullptr; + auto txPtr = getTxByKey(key); + if (txPtr == nullptr) { + throw std::runtime_error(std::format("not hash for zc key {:x}", key)); } - return getTxByKey(key); + return txPtr->getTxHash(); } +//////// TxOut MempoolSnapshot::getTxOutCopy(Types::TxKey key, Types::TxIOId outputId) const { auto txPtr = getTxByKey(key); @@ -1110,27 +1135,24 @@ std::shared_ptr MempoolSnapshot::getTxioByKey( return data_->getTxio(txioKey); } +//////// Types::TxKey MempoolSnapshot::getKeyForHash( const Types::TxHash& hash) const { return data_->getKeyForHash(hash); } -const Types::TxHash& MempoolSnapshot::getHashForKey( - Types::TxKey key) const +std::shared_ptr MempoolSnapshot::getTxByHash( + const Types::TxHash& hash) const { - auto txPtr = getTxByKey(key); - if (txPtr == nullptr) { - throw std::runtime_error(std::format("not hash for zc key {:x}", key)); + auto key = getKeyForHash(hash); + if (!Types::isTxKeyValid(key)) { + return nullptr; } - return txPtr->getTxHash(); -} - -Types::ZcId MempoolSnapshot::getTopZcID() const -{ - return topID_; + return getTxByKey(key); } +//////// bool MempoolSnapshot::hasHash(const Types::TxHash& hash) const { return data_->getKeyForHash(hash) != Types::INVALID_TX_KEY; @@ -1141,6 +1163,7 @@ bool MempoolSnapshot::isTxOutSpentByZC(Types::TxIOKey key) const return data_->isTxOutSpentByZC(key); } +//////// const TxIOKeys& MempoolSnapshot::getTxioKeysForScrAddr( const Types::ScrAddr& scrAddr) const { diff --git a/cppForSwig/ZeroConf/Utils.h b/cppForSwig/ZeroConf/Utils.h index d88d022fc..9a36467e5 100644 --- a/cppForSwig/ZeroConf/Utils.h +++ b/cppForSwig/ZeroConf/Utils.h @@ -215,6 +215,7 @@ namespace Armory void copyFrom(const MempoolData&); std::shared_ptr getTx(Types::TxKey) const; + size_t getTxCount(void) const; std::shared_ptr getTxio(Types::TxIOKey) const; Types::TxKey getKeyForHash(const Types::TxHash&) const; bool isTxOutSpentByZC(Types::TxIOKey) const; @@ -268,6 +269,7 @@ namespace Armory const Types::TxHash& getHashForKey(Types::TxKey) const; bool hasHash(const Types::TxHash&) const; + size_t getTxCount(void) const; Types::ZcId getTopZcID(void) const; bool isTxOutSpentByZC(Types::TxIOKey) const; diff --git a/cppForSwig/gtest/ZeroConfTests.cpp b/cppForSwig/gtest/ZeroConfTests.cpp index 4bc5ebbb6..0dd99aa7a 100644 --- a/cppForSwig/gtest/ZeroConfTests.cpp +++ b/cppForSwig/gtest/ZeroConfTests.cpp @@ -1353,18 +1353,23 @@ TEST_F(ZeroConfTests_Mempool, DropParent_Commit) class ZeroConfTests_FullNode : public ::testing::Test { protected: - void initBDM(void) + void initBDM(bool clearMempool=false) { - Config::reset(); - Config::DBSettings::setServiceType(SERVICE_UNITTEST); - Config::parseArgs({ + std::vector args{ "--datadir=./fakehomedir", "--dbdir=./ldbtestdir", "--satoshi-datadir=./blkfiletest", "--db-type=DB_FULL", "--thread-count=3", - "--public"}, - Config::ProcessType::DB); + "--public" + }; + if (clearMempool) { + args.push_back({"--clear-mempool"}); + } + + Config::reset(); + Config::DBSettings::setServiceType(SERVICE_UNITTEST); + Config::parseArgs(args, Config::ProcessType::DB); DBTestUtils::init(); theBDMt_ = new BlockDataManagerThread(); @@ -1450,7 +1455,7 @@ class ZeroConfTests_FullNode : public ::testing::Test }; //////////////////////////////////////////////////////////////////////////////// -TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) +TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC1) { TestUtils::setBlocks({ "0", "1", "2", "3" }, blk0dat_); clients_->init(); @@ -1572,7 +1577,10 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) //check the two ZCs are in the mempool { auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 2); auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); auto zcKey0 = ss->getKeyForHash(zcHash0); @@ -1587,8 +1595,6 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) ASSERT_TRUE(Types::isThisAZCKey(zcKey1)); auto zcId1 = Types::getZcIdFromTxKey(zcKey1); ASSERT_EQ(zcId1, 1); - - ASSERT_EQ(ss->getTopZcID(), 1); } //restart BDM @@ -1635,7 +1641,10 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) //check the two ZCs are in the mempool { auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 2); auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); auto zcKey0 = ss->getKeyForHash(zcHash0); @@ -1650,8 +1659,6 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) ASSERT_TRUE(Types::isThisAZCKey(zcKey1)); auto zcId1 = Types::getZcIdFromTxKey(zcKey1); ASSERT_EQ(zcId1, 1); - - ASSERT_EQ(ss->getTopZcID(), 1); } //add last 2 blocks @@ -1674,7 +1681,10 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) //check the two ZCs are out of the mempool { auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 0); auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); auto zcKey0 = ss->getKeyForHash(zcHash0); @@ -1685,8 +1695,6 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) auto zcKey1 = ss->getKeyForHash(zcHash1); ASSERT_FALSE(Types::isTxKeyValid(zcKey1)); ASSERT_FALSE(Types::isThisAZCKey(zcKey1)); - - ASSERT_EQ(ss->getTopZcID(), 1); } //restart BDM again @@ -1733,6 +1741,7 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) //check mempool is empty { auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 0); auto ss = zcParser->getSnapshot(); ASSERT_EQ(ss, nullptr); } @@ -1741,6 +1750,461 @@ TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC_Plus2) bdvPtr.reset(); } +TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ZC2) +{ + TestUtils::setBlocks({ "0", "1", "2", "3" }, blk0dat_); + clients_->init(); + auto bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + std::vector scrAddrVec { + TestChain::scrAddrA, + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrE + }; + + const std::vector lb1ScrAddrs { + TestChain::lb1ScrAddr, + TestChain::lb1ScrAddrP2SH + }; + const std::vector lb2ScrAddrs { + TestChain::lb2ScrAddr, + TestChain::lb2ScrAddrP2SH + }; + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + auto bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 3U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash3); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash3)->isMainBranch()); + + auto bdm = theBDMt_->bdm(); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //add ZC + std::filesystem::path zcPath(TestUtils::dataDir / "ZCtx.tx"); + BinaryData rawZC; rawZC.resize(TestChain::zcTxSize); + std::ifstream zcStream(zcPath, std::ios::in | std::ios::binary); + zcStream.read(rawZC.getCharPtr(), TestChain::zcTxSize); + zcStream.close(); + DBTestUtils::ZcVector rawZcVec; + rawZcVec.push_back(std::move(rawZC), 0); + + std::filesystem::path lbPath(TestUtils::dataDir / "LBZC.tx"); + BinaryData rawLBZC; rawLBZC.resize(TestChain::lbZCTxSize); + std::ifstream lbStream(lbPath, std::ios::in | std::ios::binary); + lbStream.read(rawLBZC.getCharPtr(), TestChain::lbZCTxSize); + lbStream.close(); + DBTestUtils::ZcVector rawLBZcVec; + rawLBZcVec.push_back(std::move(rawLBZC), 0); + + DBTestUtils::pushNewZc(theBDMt_, rawZcVec); + DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + + DBTestUtils::pushNewZc(theBDMt_, rawLBZcVec); + DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 65 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //check the two ZCs are in the mempool + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); + auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 2); + + auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); + auto zcKey0 = ss->getKeyForHash(zcHash0); + ASSERT_TRUE(Types::isTxKeyValid(zcKey0)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey0)); + auto zcId0 = Types::getZcIdFromTxKey(zcKey0); + ASSERT_EQ(zcId0, 0); + + auto zcHash1 = rawLBZcVec.zcVec_[0].first.getThisHash(); + auto zcKey1 = ss->getKeyForHash(zcHash1); + ASSERT_TRUE(Types::isTxKeyValid(zcKey1)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey1)); + auto zcId1 = Types::getZcIdFromTxKey(zcKey1); + ASSERT_EQ(zcId1, 1); + } + + //restart BDM + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + //add last 2 blocks before restart + TestUtils::setBlocks({ "0", "1", "2", "3", "4", "5" }, blk0dat_); + + initBDM(); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 5U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash5); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash5)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 25 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 0 * COIN); + + //check mempool is empty + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); + + //there are 2 zc in mempool at load, which match mined tx on chain, + //therefor zc container should have a snapshot, albeit an empty one + auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 0); + } + + //restart BDM again + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + initBDM(); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 5U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash5); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash5)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 70 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 25 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 0 * COIN); + + //check mempool is empty + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 0); + + //zc container should not have a snapshot this time, as mempool + //db should be empty + auto ss = zcParser->getSnapshot(); + ASSERT_EQ(ss, nullptr); + } +} + +TEST_F(ZeroConfTests_FullNode, Load4Blocks_ReloadBDM_ClearMempool) +{ + TestUtils::setBlocks({ "0", "1", "2", "3" }, blk0dat_); + clients_->init(); + auto bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + std::vector scrAddrVec { + TestChain::scrAddrA, + TestChain::scrAddrB, + TestChain::scrAddrC, + TestChain::scrAddrE + }; + + const std::vector lb1ScrAddrs { + TestChain::lb1ScrAddr, + TestChain::lb1ScrAddrP2SH + }; + const std::vector lb2ScrAddrs { + TestChain::lb2ScrAddr, + TestChain::lb2ScrAddrP2SH + }; + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + auto bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 3U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash3); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash3)->isMainBranch()); + + auto bdm = theBDMt_->bdm(); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //restart bdm + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + initBDM(); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //add ZC + std::filesystem::path zcPath(TestUtils::dataDir / "ZCtx.tx"); + BinaryData rawZC; rawZC.resize(TestChain::zcTxSize); + std::ifstream zcStream(zcPath, std::ios::in | std::ios::binary); + zcStream.read(rawZC.getCharPtr(), TestChain::zcTxSize); + zcStream.close(); + DBTestUtils::ZcVector rawZcVec; + rawZcVec.push_back(std::move(rawZC), 0); + + std::filesystem::path lbPath(TestUtils::dataDir / "LBZC.tx"); + BinaryData rawLBZC; rawLBZC.resize(TestChain::lbZCTxSize); + std::ifstream lbStream(lbPath, std::ios::in | std::ios::binary); + lbStream.read(rawLBZC.getCharPtr(), TestChain::lbZCTxSize); + lbStream.close(); + DBTestUtils::ZcVector rawLBZcVec; + rawLBZcVec.push_back(std::move(rawLBZC), 0); + + DBTestUtils::pushNewZc(theBDMt_, rawZcVec); + DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + + DBTestUtils::pushNewZc(theBDMt_, rawLBZcVec); + DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 20 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 65 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 5 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //check the two ZCs are in the mempool + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); + auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 2); + + auto zcHash0 = rawZcVec.zcVec_[0].first.getThisHash(); + auto zcKey0 = ss->getKeyForHash(zcHash0); + ASSERT_TRUE(Types::isTxKeyValid(zcKey0)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey0)); + auto zcId0 = Types::getZcIdFromTxKey(zcKey0); + ASSERT_EQ(zcId0, 0); + + auto zcHash1 = rawLBZcVec.zcVec_[0].first.getThisHash(); + auto zcKey1 = ss->getKeyForHash(zcHash1); + ASSERT_TRUE(Types::isTxKeyValid(zcKey1)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey1)); + auto zcId1 = Types::getZcIdFromTxKey(zcKey1); + ASSERT_EQ(zcId1, 1); + } + + //restart BDM, clear the mempool + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + initBDM(true); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 3U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash3); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash3)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //check the two ZCs are in the mempool + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 0); + auto ss = zcParser->getSnapshot(); + ASSERT_EQ(ss, nullptr); + } + + //reload BDM again, make sure mempool stays clear + bdvPtr.reset(); + clients_->shutdown(); + theBDMt_->shutdown(); + + delete clients_; + delete theBDMt_; + + initBDM(); + clients_->init(); + bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + bdm = theBDMt_->bdm(); + EXPECT_EQ(TestUtils::getTopBlockHeightInDB(theBDMt_->bdm().get(), DB_SELECT::HEADERS), 3U); + EXPECT_EQ(DBTestUtils::getTopBlockHash(iface_, DB_SELECT::HEADERS), TestChain::blkHash3); + EXPECT_TRUE(theBDMt_->bdm()->blockchain()->getHeaderByHash(TestChain::blkHash3)->isMainBranch()); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb1ScrAddrP2SH, bdm), 0 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddr, bdm), 10 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::lb2ScrAddrP2SH, bdm), 5 * COIN); + + //check the two ZCs are in the mempool + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 0); + auto ss = zcParser->getSnapshot(); + ASSERT_EQ(ss, nullptr); + } +} + //////////////////////////////////////////////////////////////////////////////// TEST_F(ZeroConfTests_FullNode, Load3Blocks_ZC_Plus3) { From 6ac52cfa5ed6d7da552dccd4d4c6e93c08581519 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sun, 27 Sep 2026 15:42:12 +0200 Subject: [PATCH 21/26] do not refeed drop ZCs to the parser --- cppForSwig/ZeroConf/Parser.cpp | 23 ++++++++++++------- cppForSwig/ZeroConf/Utils.cpp | 2 +- cppForSwig/ZeroConf/Utils.h | 2 +- cppForSwig/gtest/ZeroConfTests.cpp | 37 +++++++++++++----------------- 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/cppForSwig/ZeroConf/Parser.cpp b/cppForSwig/ZeroConf/Parser.cpp index 0afe3f81f..2424275c3 100644 --- a/cppForSwig/ZeroConf/Parser.cpp +++ b/cppForSwig/ZeroConf/Parser.cpp @@ -270,7 +270,7 @@ std::map> ZeroConfContainer::purge( - evict all the descendants of mined and invalidated ZCs - for descendants, reset all resolved spenders. - - return any descendant that wasn't invalidated (for reparsing and + - return any descendant that wasn't invalidated (for reparsing and potential reentry in the mempool) * reorgs are first handled in purgeToBranchpoint @@ -384,15 +384,15 @@ std::map> ZeroConfContainer::dropZC( { /* ZeroConfSharedSnapshot will drop the tx and its children and return them. - We need to clear our containers all dropped ZCs so we first drop from the - snapshot and use the returned map to clear the requested ZC as well as all - of its children. + We need to clear our containers from all dropped ZCs so we first drop from + the snapshot then use the returned map to clear the requested ZC as well as + all of its children. */ auto droppedZCs = ss->dropZc(key); for (const auto& zcPair : droppedZCs) { auto txPtr = zcPair.second; if (txPtr == nullptr) { - return {}; + continue; } //drop from outPointsSpentByKey_ @@ -417,6 +417,8 @@ std::map> ZeroConfContainer::dropZC( keyToFundedScrAddr_.erase(key); allZcTxHashes_.erase(txPtr->getTxHash()); } + + //remove the target key from the return set return droppedZCs; } @@ -430,7 +432,9 @@ std::map> ZeroConfContainer::dropZCs( std::map> droppedZCs; auto rIter = zcKeys.rbegin(); while (rIter != zcKeys.rend()) { - auto dropped = dropZC(ss, *rIter++); + auto keyToDrop = *rIter++; + auto dropped = dropZC(ss, keyToDrop); + dropped.erase(keyToDrop); droppedZCs.insert(dropped.begin(), dropped.end()); } @@ -489,6 +493,7 @@ void ZeroConfContainer::finalizePurgePacket( zcAction.resultPromise->set_value(purgePacket); } +//////// void ZeroConfContainer::parseNewZC(ZcActionStruct zcAction) { bool notify = true; @@ -507,7 +512,7 @@ void ZeroConfContainer::parseNewZC(ZcActionStruct zcAction) auto result = purge(zcAction.reorgState, ss); notify = false; - ss->commitNewZCs(); + ss->mergeWithParents(); //setup batch with all tracked zc if (zcAction.batch == nullptr) { @@ -724,6 +729,7 @@ void ZeroConfContainer::parseNewZC( watcherMap); } +//////// FilteredZeroConfData ZeroConfContainer::filterTransaction( std::shared_ptr parsedTx, std::shared_ptr ss) const @@ -790,6 +796,7 @@ ZeroConfContainer::checkForCollisions( return invalidatedZCs; } +//////// void ZeroConfContainer::clear() { snapshot_.store(nullptr); @@ -1011,7 +1018,7 @@ unsigned ZeroConfContainer::loadMempool(bool clearMempool) UINT64_MAX, emptyWatcherMap); auto thisSnapshot = snapshot_.load(std::memory_order_acquire); - thisSnapshot->commitNewZCs(); + thisSnapshot->mergeWithParents(); } return topId; } diff --git a/cppForSwig/ZeroConf/Utils.cpp b/cppForSwig/ZeroConf/Utils.cpp index a170ccd68..18dc385c3 100644 --- a/cppForSwig/ZeroConf/Utils.cpp +++ b/cppForSwig/ZeroConf/Utils.cpp @@ -1342,7 +1342,7 @@ std::shared_ptr MempoolSnapshot::copy( } /////////////////////////////////////////////////////////////////////////////// -void MempoolSnapshot::commitNewZCs() +void MempoolSnapshot::mergeWithParents() { //figure out depth and size of each mempool obj, merge if necessary if (data_->txioMap_.empty() && diff --git a/cppForSwig/ZeroConf/Utils.h b/cppForSwig/ZeroConf/Utils.h index 9a36467e5..a7809daea 100644 --- a/cppForSwig/ZeroConf/Utils.h +++ b/cppForSwig/ZeroConf/Utils.h @@ -277,7 +277,7 @@ namespace Armory std::map> dropZc(Types::TxKey); void stageNewZC(std::shared_ptr, const FilteredZeroConfData&); - void commitNewZCs(void); + void mergeWithParents(void); unsigned getMergeCount(void) const { return mergeCount_; } }; diff --git a/cppForSwig/gtest/ZeroConfTests.cpp b/cppForSwig/gtest/ZeroConfTests.cpp index 0dd99aa7a..389a90cc7 100644 --- a/cppForSwig/gtest/ZeroConfTests.cpp +++ b/cppForSwig/gtest/ZeroConfTests.cpp @@ -677,7 +677,7 @@ TEST_F(ZeroConfTests_Mempool, Commit) EXPECT_TRUE(checkTxIsStaged(snapshot, 0)); //commit - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); //check the tx is still in there EXPECT_TRUE(checkTxIsStaged(snapshot, 0)); @@ -739,7 +739,7 @@ TEST_F(ZeroConfTests_Mempool, CommitAndDrop) EXPECT_EQ(snapshot.getTopZcID(), 1U); //commit and check again - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_TRUE(checkTxIsStaged(snapshot, 0)); //drop the tx @@ -751,7 +751,7 @@ TEST_F(ZeroConfTests_Mempool, CommitAndDrop) EXPECT_TRUE(checkIsDropped(snapshot, 0)); //commit and check - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_TRUE(checkIsDropped(snapshot, 0)); EXPECT_EQ(snapshot.getTopZcID(), 1U); } @@ -827,7 +827,7 @@ TEST_F(ZeroConfTests_Mempool, Stage2_Commit_Drop1) EXPECT_TRUE(checkTxIsStaged(snapshot, 0)); } - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_EQ(snapshot.getTopZcID(), 1U); { @@ -844,7 +844,7 @@ TEST_F(ZeroConfTests_Mempool, Stage2_Commit_Drop1) EXPECT_TRUE(checkTxIsStaged(snapshot, 1)); } - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_EQ(snapshot.getTopZcID(), 2U); //drop tx0 @@ -857,7 +857,7 @@ TEST_F(ZeroConfTests_Mempool, Stage2_Commit_Drop1) EXPECT_TRUE(checkTxIsStaged(snapshot, 1)); EXPECT_EQ(snapshot.getTopZcID(), 2U); - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); //check it is still dropped from the snapshot EXPECT_TRUE(checkIsDropped(snapshot, 0)); @@ -873,7 +873,7 @@ TEST_F(ZeroConfTests_Mempool, Stage2_Commit_Drop1) EXPECT_TRUE(checkIsDropped(snapshot, 1)); EXPECT_EQ(snapshot.getTopZcID(), 2U); - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_EQ(snapshot.getTopZcID(), 2U); } @@ -1018,7 +1018,7 @@ TEST_F(ZeroConfTests_Mempool, StageChildren_Commit) EXPECT_EQ(checkTxOutIsSpent(snapshot, 1, 0), Types::INVALID_TXIO_KEY); EXPECT_EQ(checkTxOutIsSpent(snapshot, 1, 1), Types::INVALID_TXIO_KEY); - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_TRUE(checkTxIsStaged(snapshot, 0)); EXPECT_EQ(checkTxOutIsSpent(snapshot, 0, 0), Types::INVALID_TXIO_KEY); @@ -1067,7 +1067,7 @@ TEST_F(ZeroConfTests_Mempool, StageChildren_Commit) EXPECT_EQ(checkTxOutIsSpent(snapshot, 2, 0), Types::INVALID_TXIO_KEY); } - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_TRUE(checkTxIsStaged(snapshot, 2)); EXPECT_TRUE(checkTxIsStaged(snapshot, 3)); @@ -1103,7 +1103,7 @@ TEST_F(ZeroConfTests_Mempool, StageChildren_Commit) auto spender3 = checkTxOutIsSpent(snapshot, 2, 0); EXPECT_EQ(Types::getTxKeyFromTxIOKey(spender3), zcKeys_[4]); - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); EXPECT_TRUE(checkTxIsStaged(snapshot, 2)); EXPECT_TRUE(checkTxIsStaged(snapshot, 3)); @@ -1291,7 +1291,7 @@ TEST_F(ZeroConfTests_Mempool, DropParent_Commit) EXPECT_EQ(Types::getTxKeyFromTxIOKey(spender2), zcKeys_[3]); } - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); { EXPECT_TRUE(checkTxIsStaged(snapshot, 2)); @@ -1332,7 +1332,7 @@ TEST_F(ZeroConfTests_Mempool, DropParent_Commit) EXPECT_EQ(Types::getTxKeyFromTxIOKey(spender3), zcKeys_[3]); } - snapshot.commitNewZCs(); + snapshot.mergeWithParents(); { EXPECT_TRUE(checkIsDropped(snapshot, 0)); @@ -3202,8 +3202,8 @@ TEST_F(ZeroConfTests_FullNode, ChainZC_RBFchild_Test) } { - ////spend 27 from wlt to assetWlt's first 2 unused addresses - ////send rest back to scrAddrA + // spend 27 from wallet1 to assetWlt's first 2 unused addresses + // send rest back to scrAddrD auto spendVal = 27 * COIN; Signing::Signer signer; @@ -3316,9 +3316,6 @@ TEST_F(ZeroConfTests_FullNode, ChainZC_RBFchild_Test) { Signing::Signer signer3; - //instantiate resolver feed overloaded object - auto assetFeed = std::make_shared(assetWlt); - //get utxo list for spend value auto unspentVec = DBTestUtils::getZCUTXOs(bdm, { addrVec[0], @@ -3350,6 +3347,7 @@ TEST_F(ZeroConfTests_FullNode, ChainZC_RBFchild_Test) //sign, verify then broadcast { + auto assetFeed = std::make_shared(assetWlt); auto lock = assetWlt->lockDecryptedContainer({}); signer3.setFeed(assetFeed); signer3.sign(); @@ -3429,10 +3427,6 @@ TEST_F(ZeroConfTests_FullNode, ChainZC_RBFchild_Test) auto spendVal = 10 * COIN; Signing::Signer signer2; - //instantiate resolver feed - auto assetFeed = - std::make_shared(assetWlt); - //get utxo list for spend value auto unspentVec = DBTestUtils::getRBFUTXOs(bdm, {addrVec[0]}); @@ -3470,6 +3464,7 @@ TEST_F(ZeroConfTests_FullNode, ChainZC_RBFchild_Test) //sign, verify then broadcast { + auto assetFeed = std::make_shared(assetWlt); auto lock = assetWlt->lockDecryptedContainer({}); signer2.setFeed(assetFeed); signer2.sign(); From c32e065fd473a30401c5e6d08e40271830886f85 Mon Sep 17 00:00:00 2001 From: goatpig Date: Sun, 27 Sep 2026 15:57:27 +0200 Subject: [PATCH 22/26] fix autodb connection retry --- cppForSwig/BridgeAPI/DBSetup.cpp | 2 ++ cppForSwig/Network/WebSocketClient.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index 0dd584231..e66803c28 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -543,7 +543,9 @@ BdvPtr Armory::Bridge::setupClientConnection( //connect to db if (!bdvPtr->connectToRemote()) { //could not connect, sleep for 250ms and try again + LOGWARN << "bdvptr connection failed, retrying"; std::this_thread::sleep_for(250ms); + continue; } bdvPtr->registerWithDB( Config::BitcoinSettings::getMagicBytes().toHexStr()); diff --git a/cppForSwig/Network/WebSocketClient.cpp b/cppForSwig/Network/WebSocketClient.cpp index 9b7c1a92e..e14210165 100755 --- a/cppForSwig/Network/WebSocketClient.cpp +++ b/cppForSwig/Network/WebSocketClient.cpp @@ -367,13 +367,13 @@ int WebSocketClient::lwsServiceHandler(struct lws* wsi, { try { instance->connected_.store(false, std::memory_order_release); - if (instance->callbackPtr_ != nullptr) { - instance->callbackPtr_->disconnected(); - } try { instance->connectionReadyProm_.set_value(false); } catch (const std::future_error&) { - //promise already set, nothing to do + //promise already set, notify of disconnection + if (instance->callbackPtr_ != nullptr) { + instance->callbackPtr_->disconnected(); + } } instance->shutdown(); } catch (const LWS_Error&) {} From 5554ef1f2efe500392525ca82fb94188006bff1e Mon Sep 17 00:00:00 2001 From: goatpig Date: Wed, 30 Sep 2026 12:38:40 +0200 Subject: [PATCH 23/26] better raw socket read handling on linux do not retry db connection if the user rejects db key get rid of broken overloaded Qt dialog accept and reject arguments fix DlgSetupManager scenario mishandling --- ArmoryQt.py | 3 +- cppForSwig/AsyncClient.cpp | 11 ++++- cppForSwig/AsyncClient.h | 1 + cppForSwig/BridgeAPI/CppBridge.cpp | 2 - cppForSwig/BridgeAPI/DBSetup.cpp | 4 ++ cppForSwig/Network/SocketObject.cpp | 57 +++++++++-------------- cppForSwig/Network/WebSocketClient.cpp | 8 +++- cppForSwig/Network/WebSocketClient.h | 2 + cppForSwig/Utils/BIP150_151.cpp | 2 +- qtdialogs/DlgAddressBook.py | 6 +-- qtdialogs/DlgExportTxHistory.py | 4 +- qtdialogs/DlgRequestPayment.py | 8 ++-- qtdialogs/DlgSendBitcoins.py | 8 ++-- qtdialogs/DlgSettings.py | 4 +- qtdialogs/DlgWalletDetails.py | 12 ++--- qtdialogs/setupmanager/DlgSetupManager.py | 12 +++-- ui/CoinControlUI.py | 8 ++-- ui/MultiSigDialogs.py | 8 ++-- 18 files changed, 84 insertions(+), 76 deletions(-) diff --git a/ArmoryQt.py b/ArmoryQt.py index 4ce8592e6..555f27861 100755 --- a/ArmoryQt.py +++ b/ArmoryQt.py @@ -729,7 +729,6 @@ def msrevsign(): # Restore any main-window geometry saved in the settings file hexgeom = TheSettings.get('MainGeometry') - hexwltsz = TheSettings.get('MainWalletCols') if len(hexgeom)>0: #QtCore.QByteArray is weak sauce, have to deser the hexit on our own @@ -1491,7 +1490,7 @@ def makeWalletCopy(self, parent, wlt, copyType='Same', suffix='', changePass=Fal return True ############################################################################# - def createAction(self, txt, slot, isCheckable=False, \ + def createAction(self, txt, slot, isCheckable=False, ttip=None, iconpath=None, shortcut=None): """ Modeled from the "Rapid GUI Programming with Python and QtCore.Qt" book, page 174 diff --git a/cppForSwig/AsyncClient.cpp b/cppForSwig/AsyncClient.cpp index 278d9f811..ada7cf257 100755 --- a/cppForSwig/AsyncClient.cpp +++ b/cppForSwig/AsyncClient.cpp @@ -695,7 +695,7 @@ void BlockDataViewer::getFeeSchedule(const std::string& strategy, sock_->pushPayload(std::move(write_payload), read_payload); } -/////////////////////////////////////////////////////////////////////////////// +//////// void BlockDataViewer::setCheckServerKeyPromptLambda( const std::function& lbd) { @@ -706,6 +706,15 @@ void BlockDataViewer::setCheckServerKeyPromptLambda( wsSock->setPubkeyPromptLambda(lbd); } +bool BlockDataViewer::valid1WayServerKey() const +{ + auto wsSock = std::dynamic_pointer_cast(sock_); + if (wsSock == nullptr) { + return false; + } + return wsSock->valid1WayServerKey(); +} + /////////////////////////////////////////////////////////////////////////////// // // BtcWallet diff --git a/cppForSwig/AsyncClient.h b/cppForSwig/AsyncClient.h index 7a9f90818..e2e99943e 100755 --- a/cppForSwig/AsyncClient.h +++ b/cppForSwig/AsyncClient.h @@ -156,6 +156,7 @@ namespace AsyncClient std::pair getRekeyCount(void) const; void setCheckServerKeyPromptLambda( const std::function&); + bool valid1WayServerKey(void) const; void addPublicKey(const SecureBinaryData&, bool); //connectivity diff --git a/cppForSwig/BridgeAPI/CppBridge.cpp b/cppForSwig/BridgeAPI/CppBridge.cpp index d0eaae427..45497fee8 100755 --- a/cppForSwig/BridgeAPI/CppBridge.cpp +++ b/cppForSwig/BridgeAPI/CppBridge.cpp @@ -1402,8 +1402,6 @@ void CppBridge::connectToIp(const std::string& ip, Network::port_t port, * Peers store is ignored, a throw away public key is generated by the client * at connection time. */ - LOGINFO << "connecting to ip"; - capnp::MallocMessageBuilder message; auto fromBridge = message.initRoot(); auto reply = fromBridge.initReply(); diff --git a/cppForSwig/BridgeAPI/DBSetup.cpp b/cppForSwig/BridgeAPI/DBSetup.cpp index e66803c28..1d2da6117 100644 --- a/cppForSwig/BridgeAPI/DBSetup.cpp +++ b/cppForSwig/BridgeAPI/DBSetup.cpp @@ -542,6 +542,10 @@ BdvPtr Armory::Bridge::setupClientConnection( //connect to db if (!bdvPtr->connectToRemote()) { + if (oneWayAuth && !bdvPtr->valid1WayServerKey()) { + //user rejected server key, abort + break; + } //could not connect, sleep for 250ms and try again LOGWARN << "bdvptr connection failed, retrying"; std::this_thread::sleep_for(250ms); diff --git a/cppForSwig/Network/SocketObject.cpp b/cppForSwig/Network/SocketObject.cpp index 0951ac9ff..4c5e061a0 100644 --- a/cppForSwig/Network/SocketObject.cpp +++ b/cppForSwig/Network/SocketObject.cpp @@ -23,6 +23,8 @@ using namespace Armory::Network; #ifdef _WIN32 //i dont know how to get linkage for this with MSYS2 halp T_T char *gai_strerrorA(int) { return nullptr; } +#else +#include #endif /////////////////////////////////////////////////////////////////////////////// @@ -317,7 +319,6 @@ bool PersistentSocket::testConnection() #ifndef _WIN32 void PersistentSocket::socketService_nix() { - int readIncrement = 8192; int timeout = 100; std::stringstream errorss; std::exception_ptr exceptptr = nullptr; @@ -363,15 +364,14 @@ void PersistentSocket::socketService_nix() } }; - bool loop = true; - while (loop) { + while (true) { auto status = poll(pfd, 2, timeout); if (status == 0) { continue; } else if (status == -1) { //poll error, process and exit loop auto errornum = errno; - LOGERR << "poll() error in readFromSocketThread: " << errornum; + LOGERR << "poll() error in socketService_nix: " << errornum; break; } @@ -389,50 +389,39 @@ void PersistentSocket::socketService_nix() } if (pfd[1].revents & POLLNVAL) { - LOGERR << "POLLNVAL in readFromSocketThread"; + LOGERR << "POLLNVAL in socketService_nix"; } //exceptions if (pfd[1].revents & POLLERR) { //break out of poll loop - LOGERR << "POLLERR error in readFromSocketThread"; + LOGERR << "POLLERR error in socketService_nix"; break; } if (pfd[1].revents & POLLIN) { - //read socket - std::vector readdata; - readdata.resize(readIncrement); - - size_t totalread = 0; + //get amount of data buffered by socket int readAmt; - - while (true) { - readAmt = recv(sockfd_, - (char*)&readdata[0] + totalread, - readIncrement, 0 - ); - - if (readAmt <= 0) { - auto errornum = errno; - if (errornum == EAGAIN || errornum == EWOULDBLOCK) { - break; - } - LOGERR << "recv error: " << errornum << ", aborting"; - loop = false; + if (ioctl(sockfd_, FIONREAD, &readAmt) == 0) { + if (readAmt == 0) { + //pollin notified socket is ready to read but buffer is empty + //treat as socket has been cleaned up + LOGDEBUG << "socket cleaned up"; break; } - - totalread += readAmt; - if (readAmt < readIncrement) { + //read it and push it into the queue + std::vector readdata; + readdata.resize(readAmt); + int totalRead = recv(sockfd_, &readdata[0], readAmt, 0); + if (totalRead == readAmt) { + readQueue_.push_back(std::move(readdata)); + } else { + LOGWARN << std::format( + "failed to read socket data with error: {}! dropping socket", errno); break; } - readdata.resize(totalread + readIncrement); - } - - if (totalread > 0) { - readdata.resize(totalread); - readQueue_.push_back(move(readdata)); + } else { + LOGWARN << "ioctl failed with error: " << errno; } } diff --git a/cppForSwig/Network/WebSocketClient.cpp b/cppForSwig/Network/WebSocketClient.cpp index e14210165..dcfc1b227 100755 --- a/cppForSwig/Network/WebSocketClient.cpp +++ b/cppForSwig/Network/WebSocketClient.cpp @@ -545,7 +545,7 @@ bool WebSocketClient::processAEADHandshake(const WebSocketMessagePartial& msgObj if (serverPubkeyProm_ != nullptr) { //wait on server pubkey announce ACK/nACK auto fut = serverPubkeyProm_->get_future(); - fut.wait(); + valid1WayServerKey_ = fut.get(); serverPubkeyProm_.reset(); } @@ -569,7 +569,6 @@ bool WebSocketClient::processAEADHandshake(const WebSocketMessagePartial& msgObj serverPubkeyProm_ = std::make_shared>(); promptUser(msgbdr, servName_); } - return true; } @@ -633,6 +632,11 @@ void WebSocketClient::setPubkeyPromptLambda( userPromptLambda_ = lbd; } +bool WebSocketClient::valid1WayServerKey() const +{ + return valid1WayServerKey_; +} + void WebSocketClient::promptUser( const BinaryDataRef& keyRef, const std::string& name) { diff --git a/cppForSwig/Network/WebSocketClient.h b/cppForSwig/Network/WebSocketClient.h index cce36318b..f3e4bef00 100644 --- a/cppForSwig/Network/WebSocketClient.h +++ b/cppForSwig/Network/WebSocketClient.h @@ -124,6 +124,7 @@ namespace Armory BinaryData leftOverData_; std::shared_ptr> serverPubkeyProm_; + bool valid1WayServerKey_ = false; std::function userPromptLambda_; public: @@ -151,6 +152,7 @@ namespace Armory std::pair getRekeyCount(void) const; void addPublicKey(const SecureBinaryData&, bool); void setPubkeyPromptLambda(const std::function&); + bool valid1WayServerKey(void) const; //virtuals SocketType type(void) const override; diff --git a/cppForSwig/Utils/BIP150_151.cpp b/cppForSwig/Utils/BIP150_151.cpp index 474f4f792..91ae3b43c 100755 --- a/cppForSwig/Utils/BIP150_151.cpp +++ b/cppForSwig/Utils/BIP150_151.cpp @@ -1145,8 +1145,8 @@ int BIP150StateMachine::getAuthreplyData(BinaryData& output, bool responderSent) Client side 1-way auth: return own pubkey */ auto ownPubKey = peerView_->getPubKeyRef("own"); - LOGERR << "BIP 150 - own public key is not 33 bytes long"; if (ownPubKey.getSize() != BIP151PUBKEYSIZE) { + LOGERR << "BIP 150 - own public key is not 33 bytes long"; return errorSM(retVal); } std::memcpy(output.getPtr(), ownPubKey.getPtr(), BIP151PUBKEYSIZE); diff --git a/qtdialogs/DlgAddressBook.py b/qtdialogs/DlgAddressBook.py index 806b7a269..fe5670a2d 100644 --- a/qtdialogs/DlgAddressBook.py +++ b/qtdialogs/DlgAddressBook.py @@ -285,12 +285,12 @@ def closeEvent(self, event): super(DlgAddressBook, self).closeEvent(event) ############################################################################# - def accept(self, *args): + def accept(self): self.saveGeometrySettings() - super(DlgAddressBook, self).accept(*args) + super(DlgAddressBook, self).accept() ############################################################################# - def reject(self, *args): + def reject(self): self.saveGeometrySettings() super(DlgAddressBook, self).reject(*args) diff --git a/qtdialogs/DlgExportTxHistory.py b/qtdialogs/DlgExportTxHistory.py index fa1ed0c06..f0b9b18ba 100644 --- a/qtdialogs/DlgExportTxHistory.py +++ b/qtdialogs/DlgExportTxHistory.py @@ -147,9 +147,9 @@ def doExampleDate(self, qstr=None): self.isValidFormat = False ############################################################################# - def accept(self, *args): + def accept(self): if self.createFile_CSV(): - super(DlgExportTxHistory, self).accept(*args) + super(DlgExportTxHistory, self).accept() ############################################################################# diff --git a/qtdialogs/DlgRequestPayment.py b/qtdialogs/DlgRequestPayment.py index 8eb620d50..1a274f199 100644 --- a/qtdialogs/DlgRequestPayment.py +++ b/qtdialogs/DlgRequestPayment.py @@ -268,15 +268,15 @@ def closeEvent(self, event): super(DlgRequestPayment, self).closeEvent(event) ############################################################################# - def accept(self, *args): + def accept(self): self.saveGeometrySettings() self.saveLinkText() - super(DlgRequestPayment, self).accept(*args) + super(DlgRequestPayment, self).accept() ############################################################################# - def reject(self, *args): + def reject(self): self.saveGeometrySettings() - super(DlgRequestPayment, self).reject(*args) + super(DlgRequestPayment, self).reject() ############################################################################# diff --git a/qtdialogs/DlgSendBitcoins.py b/qtdialogs/DlgSendBitcoins.py index fbee44891..01d076b46 100644 --- a/qtdialogs/DlgSendBitcoins.py +++ b/qtdialogs/DlgSendBitcoins.py @@ -66,11 +66,11 @@ def closeEvent(self, event): super(DlgSendBitcoins, self).closeEvent(event) ############################################################################# - def accept(self, *args): + def accept(self): self.saveGeometrySettings() - super(DlgSendBitcoins, self).accept(*args) + super(DlgSendBitcoins, self).accept() ############################################################################# - def reject(self, *args): + def reject(self): self.saveGeometrySettings() - super(DlgSendBitcoins, self).reject(*args) \ No newline at end of file + super(DlgSendBitcoins, self).reject() \ No newline at end of file diff --git a/qtdialogs/DlgSettings.py b/qtdialogs/DlgSettings.py index 8e8dda8f0..54c1c8527 100644 --- a/qtdialogs/DlgSettings.py +++ b/qtdialogs/DlgSettings.py @@ -634,7 +634,7 @@ def setAddrType(addrType): self.frmAddrType.setLayout(frmAddrLayout) ############################################################################# - def accept(self, *args): + def accept(self): if self.chkManageSatoshi.isChecked(): # Check valid path is supplied for bitcoin installation pathExe = str(self.edtSatoshiExePath.text()).strip() @@ -729,7 +729,7 @@ def accept(self, *args): self.main.createCombinedLedger() except: pass - super(DlgSettings, self).accept(*args) + super(DlgSettings, self).accept() ############################################################################# def setUsermodeDescr(self): diff --git a/qtdialogs/DlgWalletDetails.py b/qtdialogs/DlgWalletDetails.py index d5f8b40b0..720b98e7f 100644 --- a/qtdialogs/DlgWalletDetails.py +++ b/qtdialogs/DlgWalletDetails.py @@ -350,14 +350,14 @@ def closeEvent(self, event): super(DlgWalletDetails, self).closeEvent(event) ############################################################################# - def accept(self, *args): + def accept(self): self.saveGeometrySettings() - super(DlgWalletDetails, self).accept(*args) + super(DlgWalletDetails, self).accept() ############################################################################# - def reject(self, *args): + def reject(self): self.saveGeometrySettings() - super(DlgWalletDetails, self).reject(*args) + super(DlgWalletDetails, self).reject() ############################################################################# def showContextMenu(self, pos): @@ -1005,7 +1005,7 @@ def __init__(self, currName='', currDescr='', parent=None, main=None): self.setWindowTitle(self.tr('Wallet Descriptions')) - def accept(self, *args): + def accept(self): try: self.edtName.text().encode("ascii") except UnicodeDecodeError: @@ -1016,4 +1016,4 @@ def accept(self, *args): QtWidgets.QMessageBox.critical(self, self.tr('Empty Name'), \ self.tr('All wallets must have a name. '), QtWidgets.QMessageBox.Ok) return - super(DlgChangeLabels, self).accept(*args) + super(DlgChangeLabels, self).accept() diff --git a/qtdialogs/setupmanager/DlgSetupManager.py b/qtdialogs/setupmanager/DlgSetupManager.py index 6f10b3b8c..a334877ed 100644 --- a/qtdialogs/setupmanager/DlgSetupManager.py +++ b/qtdialogs/setupmanager/DlgSetupManager.py @@ -445,7 +445,7 @@ def saveSettings(self): self._setSettingIfChanged( 'RemoteIpAddr', dbSettings['ipAddr']) self._setSettingIfChanged( - 'RemoteIpPort', dbSettings['ipPort']) + 'RemoteIpPort', dbSettings['dbPort']) elif dbScenario == SCENARIO_REMOTE_PEER: self._setSettingIfChanged( 'RemotePeerKey', dbSettings['peerKey']) @@ -502,11 +502,13 @@ def initiateDbConnection(self, params=None): elif scenario == SCENARIO_REMOTE_PEER: success, error = self._connectToPeer(params) self._handleConnectionAttemptFinality(success, error) + return success, error elif scenario == SCENARIO_REMOTE_IP: success, error = self._connectToIp(params) self._handleConnectionAttemptFinality(success, error) - - raise ValueError(f"Unknown scenario: {scenario}") + return success, error + else: + return False, f"Unknown scenario: {scenario}" def _handleAutomationReply(self, reply): TheSignalExecution.executeMethod( @@ -569,8 +571,8 @@ def _connectToIp(self, params): 5. Result callback fires with actual success/failure """ dbAddr = params.get('ipAddr', '') - dbPort = params.get( - 'dbPort', ARMORYDB_DEFAULT_PORT) + dbPort = int(params.get( + 'dbPort', ARMORYDB_DEFAULT_PORT)) if not dbAddr: raise ValueError( diff --git a/ui/CoinControlUI.py b/ui/CoinControlUI.py index c017ec80a..18f4825a3 100755 --- a/ui/CoinControlUI.py +++ b/ui/CoinControlUI.py @@ -111,9 +111,9 @@ def closeEvent(self, event): super(CoinControlDlg, self).closeEvent(event) ############################################################################# - def accept(self, *args): + def accept(self): self.saveGeometrySettings() - super(CoinControlDlg, self).accept(*args) + super(CoinControlDlg, self).accept() ############################################################################# def resetTreeData(self): @@ -121,9 +121,9 @@ def resetTreeData(self): self.ccTreeModel.reset() ############################################################################# - def reject(self, *args): + def reject(self): self.saveGeometrySettings() - super(CoinControlDlg, self).reject(*args) + super(CoinControlDlg, self).reject() ############################################################################# def saveGeometrySettings(self): diff --git a/ui/MultiSigDialogs.py b/ui/MultiSigDialogs.py index e43636b35..39f650263 100644 --- a/ui/MultiSigDialogs.py +++ b/ui/MultiSigDialogs.py @@ -1813,16 +1813,16 @@ def closeEvent(self, event): super(DlgLockboxManager, self).closeEvent(event) ############################################################################# - def accept(self, *args): + def accept(self): self.saveGeometrySettings() self.main.lbDialogModel = None - super(DlgLockboxManager, self).accept(*args) + super(DlgLockboxManager, self).accept() ############################################################################# - def reject(self, *args): + def reject(self): self.saveGeometrySettings() self.main.lbDialogModel = None - super(DlgLockboxManager, self).reject(*args) + super(DlgLockboxManager, self).reject() ############################################################################# def changeLBFilter(self): From 482c9807e99bf4d25e1fb709fb6d926f9132d481 Mon Sep 17 00:00:00 2001 From: goatpig Date: Wed, 30 Sep 2026 14:51:39 +0200 Subject: [PATCH 24/26] do not track unrelated ZC in db bare/full --- cppForSwig/ZeroConf/Parser.cpp | 7 +- cppForSwig/gtest/ZeroConfTests.cpp | 184 ++++++++++++++++++++++++++++- 2 files changed, 185 insertions(+), 6 deletions(-) diff --git a/cppForSwig/ZeroConf/Parser.cpp b/cppForSwig/ZeroConf/Parser.cpp index 2424275c3..6b31106f8 100644 --- a/cppForSwig/ZeroConf/Parser.cpp +++ b/cppForSwig/ZeroConf/Parser.cpp @@ -654,12 +654,12 @@ void ZeroConfContainer::parseNewZC( continue; } //in bare/full node, zcs that cannot be resolved do not affect - //our list of addresses, drop them - droppedZcKeys.emplace(newZCPair.first); + //our list of addresses, do not track them + batch.zcToWrite.erase(newZCPair.first); } } - //get rid of invalid zc, only applies to bare/full node + //get rid of invalid/mined zc dropZCs(ss, droppedZcKeys); if (updateDB && batch.hasData()) { @@ -904,7 +904,6 @@ void ZeroConfContainer::updateZCinDB(ZcUpdateBatch batch) } for (const auto& txhash : batch.txHashes) { - //if the key is not to be found in the txMap_, this is a ZC txhash tx->insert( LMDB::DataRef{txhash.getSize(), txhash.getPtr()}, LMDB::DataRef{0, (const char*)nullptr} diff --git a/cppForSwig/gtest/ZeroConfTests.cpp b/cppForSwig/gtest/ZeroConfTests.cpp index 389a90cc7..e751550dc 100644 --- a/cppForSwig/gtest/ZeroConfTests.cpp +++ b/cppForSwig/gtest/ZeroConfTests.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -2477,7 +2478,7 @@ TEST_F(ZeroConfTests_FullNode, Load3Blocks_RBF) bw.put_uint32_t(1); //flagged sequence number //spend script, classic P2PKH - BinaryData fakeAddr = + BinaryData fakeAddr = READHEX("0101010101010101010101010101010101010101"); BinaryWriter spendScript; spendScript.put_uint8_t(OP_DUP); @@ -3816,7 +3817,6 @@ TEST_F(ZeroConfTests_FullNode, PoisonTxHints) bdvPtr.reset(); } -//////////////////////////////////////////////////////////////////////////////// //add a zc, add a txhint for that hash that points to a valid mined tx, //then try to spend from first zc TEST_F(ZeroConfTests_FullNode, PoisonTxHints2) @@ -4119,6 +4119,186 @@ TEST_F(ZeroConfTests_FullNode, PoisonTxHints2) bdvPtr.reset(); } +//////////////////////////////////////////////////////////////////////////////// +// zc that don't affect the registered wallets in bare/full node should +// not be saved in db +TEST_F(ZeroConfTests_FullNode, DBBloat) +{ + BinaryData mockZC1, mockZC2; + + { + //build bogus zc + BinaryWriter bw; + bw.put_uint32_t(1); //version number + + //input + bw.put_var_int(1); //1 input, no need to complicate this + bw.put_BinaryData(Cryptography::PRNG::fortuna.generateRandom(32)); //hash of tx we are spending + bw.put_uint32_t(0); //output id + bw.put_var_int(0); //empty script, zc parser does not check sigs at any rate + bw.put_uint32_t(1); //flagged sequence number + + //spend script, classic P2PKH + BinaryData fakeAddr = + READHEX("0101010101010101010101010101010101010101"); + BinaryWriter spendScript; + spendScript.put_uint8_t(OP_DUP); + spendScript.put_uint8_t(OP_HASH160); + spendScript.put_var_int(fakeAddr.getSize()); + spendScript.put_BinaryData(fakeAddr); //bogus address + spendScript.put_uint8_t(OP_EQUALVERIFY); + spendScript.put_uint8_t(OP_CHECKSIG); + + auto& spendScriptbd = spendScript.getData(); + + //output + bw.put_var_int(1); //txout count + bw.put_uint64_t(30 * COIN); //value + bw.put_var_int(spendScriptbd.getSize()); //script length + bw.put_BinaryData(spendScriptbd); //spend script + + //locktime + bw.put_uint32_t(UINT32_MAX); + + mockZC1 = bw.getData(); + } + + { + //build another bogus ZC + BinaryWriter bw; + bw.put_uint32_t(1); //version number + + //input + bw.put_var_int(1); + bw.put_BinaryData(Cryptography::PRNG::fortuna.generateRandom(32)); + bw.put_uint32_t(3); + bw.put_var_int(0); + bw.put_uint32_t(1); + + //spend script, classic P2PKH + BinaryWriter spendScript; + spendScript.put_uint8_t(OP_DUP); + spendScript.put_uint8_t(OP_HASH160); + spendScript.put_var_int(TestChain::addrA.getSize()); + spendScript.put_BinaryData(TestChain::addrA); //spend back to self + spendScript.put_uint8_t(OP_EQUALVERIFY); + spendScript.put_uint8_t(OP_CHECKSIG); + + auto& spendScriptbd = spendScript.getData(); + + //output + bw.put_var_int(1); + bw.put_uint64_t(11 * COIN); //value + bw.put_var_int(spendScriptbd.getSize()); //script length + bw.put_BinaryData(spendScriptbd); //spend script + + //locktime + bw.put_uint32_t(UINT32_MAX); + mockZC2 = bw.getData(); + } + + auto mockZC1Hash = BtcUtils::getHash256(mockZC1); + auto mockZC2Hash = BtcUtils::getHash256(mockZC2); + + DBTestUtils::ZcVector zcVec; + zcVec.push_back(mockZC1, 0, 0); + zcVec.push_back(mockZC2, 0, 0); + + //copy the first 4 blocks + TestUtils::setBlocks({ "0", "1", "2", "3" }, blk0dat_); + clients_->init(); + theBDMt_->start(Config::DBSettings::initMode()); + theBDMt_->bdm()->blockUntilReady(); + auto bdvID = DBTestUtils::registerBDV(clients_, Config::BitcoinSettings::getMagicBytes()); + + std::vector scrAddrVec { + TestChain::scrAddrA, + TestChain::scrAddrB, + TestChain::scrAddrC + }; + + const std::vector lb1ScrAddrs { + TestChain::lb1ScrAddr, + TestChain::lb1ScrAddrP2SH + }; + const std::vector lb2ScrAddrs { + TestChain::lb2ScrAddr, + TestChain::lb2ScrAddrP2SH + }; + + DBTestUtils::registerWallet(clients_, bdvID, scrAddrVec, "wallet1", + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb1ScrAddrs, TestChain::lb1B58ID, + false); + DBTestUtils::registerWallet( + clients_, bdvID, lb2ScrAddrs, TestChain::lb2B58ID, + false); + auto bdvPtr = DBTestUtils::getBDV(clients_, bdvID); + + //wait on signals + DBTestUtils::goOnline(clients_, bdvID); + DBTestUtils::waitOnBDVReady(clients_, bdvID); + + auto bdm = theBDMt_->bdm(); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 50 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + + //push the 2 ZC + DBTestUtils::pushNewZc(theBDMt_, zcVec); + DBTestUtils::waitOnNewZcSignal(clients_, bdvID); + + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrA, bdm), 61 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrB, bdm), 30 * COIN); + EXPECT_EQ(DBTestUtils::getScrAddrBalance(TestChain::scrAddrC, bdm), 55 * COIN); + + //check mempool + Types::ZcId zc2Id; + { + auto zcParser = bdm->zeroConfCont(); + ASSERT_EQ(zcParser->getTopZcID(), 2); + auto ss = zcParser->getSnapshot(); + ASSERT_NE(ss, nullptr); + ASSERT_EQ(ss->getTxCount(), 1); + + auto zcKey1 = ss->getKeyForHash(mockZC1Hash); + ASSERT_FALSE(Types::isTxKeyValid(zcKey1)); + ASSERT_FALSE(Types::isThisAZCKey(zcKey1)); + + auto zcKey2 = ss->getKeyForHash(mockZC2Hash); + ASSERT_TRUE(Types::isTxKeyValid(zcKey2)); + ASSERT_TRUE(Types::isThisAZCKey(zcKey2)); + zc2Id = Types::getZcIdFromTxKey(zcKey2); + ASSERT_EQ(zc2Id, 1); + } + + //check db + { + auto db = bdm->getIFace(); + auto tx = db->beginTransaction(DB_SELECT::ZERO_CONF, LMDB::Mode::ReadOnly); + + //zc db should have entries + auto dbIter = tx->getIterator(); + ASSERT_TRUE(dbIter.seekToStartsWith(DbPrefix::ZCDATA)); + + //check all keys in zc db + do { + auto keyRef = dbIter.getKeyRef(); + if (keyRef.getSize() == 9) { + Types::TxKey zckey; + std::memcpy(&zckey, keyRef.getPtr() + 1, sizeof(Types::TxKey)); + ASSERT_TRUE(Types::isThisAZCKey(zckey)); + auto zcId = Types::getZcIdFromTxKey(zckey); + ASSERT_EQ(zcId, zc2Id); + } else { + ASSERT_EQ(keyRef.getSize(), 32); + ASSERT_EQ(keyRef, mockZC2Hash); + } + } while (dbIter.advanceAndRead(DbPrefix::ZCDATA)); + } +} + //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //NOTE: have to rework supernode scanner before getting to this From 3518a159ae72a355fd57f1615bac42fe2d34946a Mon Sep 17 00:00:00 2001 From: goatpig Date: Thu, 1 Oct 2026 09:25:14 +0200 Subject: [PATCH 25/26] fix bdm ready handling --- cppForSwig/BDM_mainthread.cpp | 28 +++++++------------- cppForSwig/BDM_mainthread.h | 8 +++--- cppForSwig/BlockchainDatabase/BlockUtils.cpp | 9 +++++-- cppForSwig/BlockchainDatabase/BlockUtils.h | 2 +- 4 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cppForSwig/BDM_mainthread.cpp b/cppForSwig/BDM_mainthread.cpp index cd6ece0b3..7c1ec621a 100644 --- a/cppForSwig/BDM_mainthread.cpp +++ b/cppForSwig/BDM_mainthread.cpp @@ -49,7 +49,12 @@ void BlockDataManagerThread::start(BdmInitMode mode) { pimpl->mode = mode; pimpl->run = true; - pimpl->tID = std::thread(thrun, this); + std::promise isReadyPromise; + pimpl->bdm->isReadyFuture = isReadyPromise.get_future(); + pimpl->tID = std::thread( + [this](std::promise prom){ this->run(std::move(prom)); }, + std::move(isReadyPromise) + ); } std::shared_ptr BlockDataManagerThread::bdm() @@ -87,8 +92,8 @@ void BlockDataManagerThread::join() } } -void BlockDataManagerThread::run() -try { +void BlockDataManagerThread::run(std::promise isReadyPromise) +{ const auto bdm = this->bdm(); try { if (bdm->hasException()) { @@ -100,9 +105,6 @@ try { return; } - std::promise isReadyPromise; - bdm->isReadyFuture = isReadyPromise.get_future(); - auto updateNodeStatusLambda = [bdm]()->void { try { @@ -164,7 +166,7 @@ try { bdm->enableZeroConf(DBSettings::clearMempool()); } } - isReadyPromise.set_value(true); + isReadyPromise.set_value(); if (DBSettings::checkChain()) { return; @@ -245,16 +247,4 @@ try { break; } } -} catch (const std::exception &e) { - LOGERR << "BDM thread failed: " << e.what(); -} catch (...) { - LOGERR << "BDM thread failed: (unknown exception)"; -} - -void* BlockDataManagerThread::thrun(void *_self) -{ - BlockDataManagerThread *const self - = static_cast(_self); - self->run(); - return 0; } diff --git a/cppForSwig/BDM_mainthread.h b/cppForSwig/BDM_mainthread.h index f21d12c42..0debbe36c 100644 --- a/cppForSwig/BDM_mainthread.h +++ b/cppForSwig/BDM_mainthread.h @@ -17,6 +17,7 @@ #include #include #include +#include #include "bdmenums.h" struct BlockDataManagerConfig; @@ -56,12 +57,11 @@ class BlockDataManagerThread std::shared_ptr bdm(void); // return true if the caller should wait on callback notification - bool shutdown(); - void join(); + bool shutdown(void); + void join(void); private: - static void* thrun(void *); - void run(); + void run(std::promise); private: BlockDataManagerThread(const BlockDataManagerThread&); diff --git a/cppForSwig/BlockchainDatabase/BlockUtils.cpp b/cppForSwig/BlockchainDatabase/BlockUtils.cpp index 87f0df77f..770bb4924 100644 --- a/cppForSwig/BlockchainDatabase/BlockUtils.cpp +++ b/cppForSwig/BlockchainDatabase/BlockUtils.cpp @@ -270,7 +270,7 @@ bool BlockDataManager::loadDiskState(const ProgressCallback &progress) BDMstate_.store((int)BDMState::Ready, std::memory_order_relaxed); readyProm.set_value(true); - LOGINFO << "BDM is ready"; + LOGINFO << "BDM is ready, top block: " << blockchain_->top()->getBlockHeight(); return true; } @@ -367,9 +367,14 @@ std::shared_ptr BlockDataManager::getNodeStatus() const //////// void BlockDataManager::blockUntilReady() const { + //for thread safety, shared_future should be copied before use + auto futCopy = isReadyFuture; while (true) { try { - isReadyFuture.wait(); + if (!futCopy.valid()) { + throw std::future_error(std::future_errc::no_state); + } + futCopy.wait(); return; } catch (const std::future_error&) { std::this_thread::sleep_for(100ms); diff --git a/cppForSwig/BlockchainDatabase/BlockUtils.h b/cppForSwig/BlockchainDatabase/BlockUtils.h index a9345cb42..3e66d3ea7 100644 --- a/cppForSwig/BlockchainDatabase/BlockUtils.h +++ b/cppForSwig/BlockchainDatabase/BlockUtils.h @@ -110,7 +110,7 @@ class BlockDataManager public: typedef std::function ProgressCallback; std::shared_ptr processNode, watchNode; - std::shared_future isReadyFuture; + std::shared_future isReadyFuture; mutable std::shared_ptr nodeRPC; Armory::Threading::TimedQueue> notificationStack; From 4aacbeac6c7601b77987bdd324dae6126595b84a Mon Sep 17 00:00:00 2001 From: goatpig Date: Thu, 1 Oct 2026 09:31:11 +0200 Subject: [PATCH 26/26] name sockets, improve verbose for socket errors/lifetime default to loginfo for db and bridge --- .../BlockchainDatabase/DatabaseBuilder.cpp | 8 +- cppForSwig/BridgeAPI/BridgeMain.cpp | 2 +- cppForSwig/BridgeAPI/BridgeSocket.cpp | 4 +- cppForSwig/Network/SocketObject.cpp | 239 +++++++++--------- cppForSwig/Network/SocketObject.h | 46 ++-- cppForSwig/Network/StringSockets.cpp | 8 +- cppForSwig/Network/StringSockets.h | 2 +- cppForSwig/Network/WebSocketClient.cpp | 66 +++-- cppForSwig/Node/BitcoinP2P.cpp | 56 ++-- cppForSwig/Node/BitcoinP2P.h | 13 +- cppForSwig/Node/nodeRPC.cpp | 10 +- cppForSwig/Utils/ArmoryConfig.cpp | 4 +- cppForSwig/main.cpp | 2 +- 13 files changed, 251 insertions(+), 209 deletions(-) diff --git a/cppForSwig/BlockchainDatabase/DatabaseBuilder.cpp b/cppForSwig/BlockchainDatabase/DatabaseBuilder.cpp index 639bab1bd..19e6b5707 100644 --- a/cppForSwig/BlockchainDatabase/DatabaseBuilder.cpp +++ b/cppForSwig/BlockchainDatabase/DatabaseBuilder.cpp @@ -234,9 +234,11 @@ namespace { } totalWriteCount += localWriteCount; auto hintsInMem = totalHintsInMemory.load(std::memory_order_relaxed); - std::cout << "wrote " << localWriteCount << " hints in " << - elapsed << " - total: " << totalWriteCount << - "/" << totalHints << ", in mem: " << hintsInMem << std::endl; + LOGDEBUG << std::format( + "wrote {} hints in {} - total: {}/{}, in mem: {}", + localWriteCount, elapsed, + totalWriteCount, totalHints, + hintsInMem); return hintsInMem > (256 * writeThreshold / 2); } } diff --git a/cppForSwig/BridgeAPI/BridgeMain.cpp b/cppForSwig/BridgeAPI/BridgeMain.cpp index 63c215260..2fd2bad0a 100644 --- a/cppForSwig/BridgeAPI/BridgeMain.cpp +++ b/cppForSwig/BridgeAPI/BridgeMain.cpp @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) //turn on logging auto bridgeLogPath = Armory::Config::Pathing::logFilePath("bridgeLog"); - STARTLOGGING(bridgeLogPath, LogLvlDebug); + STARTLOGGING(bridgeLogPath, LogLvlInfo); LOGENABLESTDOUT(); LOGINFO << "bridge log: " << bridgeLogPath.string(); diff --git a/cppForSwig/BridgeAPI/BridgeSocket.cpp b/cppForSwig/BridgeAPI/BridgeSocket.cpp index 49af4488b..cc95ada88 100755 --- a/cppForSwig/BridgeAPI/BridgeSocket.cpp +++ b/cppForSwig/BridgeAPI/BridgeSocket.cpp @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // // -// Copyright (C) 2019-2025, goatpig // +// Copyright (C) 2019-2026, goatpig // // Distributed under the MIT license // // See LICENSE-MIT or https://opensource.org/licenses/MIT // // // @@ -30,7 +30,7 @@ using namespace Armory::Bridge; CppBridgeSocket::CppBridgeSocket( const std::string& addr, Network::port_t port, std::shared_ptr bridgePtr) : - PersistentSocket(addr, port), bridgePtr_(bridgePtr), + PersistentSocket(addr, port, "bridge_socket"), bridgePtr_(bridgePtr), serverName_(std::format("{}:{}", addr, port)) { //setup auth peers db diff --git a/cppForSwig/Network/SocketObject.cpp b/cppForSwig/Network/SocketObject.cpp index 4c5e061a0..e594cdc0a 100644 --- a/cppForSwig/Network/SocketObject.cpp +++ b/cppForSwig/Network/SocketObject.cpp @@ -32,17 +32,18 @@ char *gai_strerrorA(int) { return nullptr; } //// SocketPrototype // /////////////////////////////////////////////////////////////////////////////// -SocketPrototype::SocketPrototype() : - addr_(""), port_(UINT16_MAX) +SocketPrototype::SocketPrototype(const std::string& name) : + addr_(""), port_(UINT16_MAX), name_(name) {} //////// SocketPrototype::SocketPrototype(const std::string& addr, - port_t port, bool doInit) : - addr_(addr), port_(port) + port_t port, const std::string& name, bool doInit) : + addr_(addr), port_(port), name_(name) { if (addr.empty() || port == 0 || port == UINT16_MAX) { - throw std::runtime_error("invalid addr/port"); + throw std::runtime_error(std::format( + "invalid addr/port for {}", name_)); } if (doInit) { @@ -50,16 +51,9 @@ SocketPrototype::SocketPrototype(const std::string& addr, } } -//////// SocketPrototype::~SocketPrototype() {} -/////////////////////////////////////////////////////////////////////////////// -bool SocketPrototype::isBlocking() const -{ - return blocking_; -} - //////// const std::string& SocketPrototype::getAddrStr() const { @@ -67,12 +61,6 @@ const std::string& SocketPrototype::getAddrStr() const } //////// -bool SocketPrototype::running() const -{ - return true; -} - -/////////////////////////////////////////////////////////////////////////////// void SocketPrototype::init() { //resolve address @@ -86,7 +74,7 @@ void SocketPrototype::init() #ifdef _WIN32 //somehow getaddrinfo doesnt handle localhost on Windows std::string addrstr = addr_; - if(addr_ == "localhost") { + if (addr_ == "localhost") { addrstr = "127.0.0.1"; } #else @@ -97,28 +85,30 @@ void SocketPrototype::init() getaddrinfo(addrstr.c_str(), portStr.c_str(), &hints, &result); for (auto ptr = result; ptr != nullptr; ptr = ptr->ai_next) { if (ptr->ai_family == AF_INET) { - memcpy(&serv_addr_, ptr->ai_addr, sizeof(sockaddr_in)); - memcpy(&serv_addr_.sa_data, &ptr->ai_addr->sa_data, 14); + std::memcpy(&serv_addr_, ptr->ai_addr, sizeof(sockaddr_in)); + std::memcpy(&serv_addr_.sa_data, &ptr->ai_addr->sa_data, 14); break; } - throw std::runtime_error("unsupported remote address format"); + throw std::runtime_error(std::format( + "unsupported remote address format for {}", name_)); } freeaddrinfo(result); } -/////////////////////////////////////////////////////////////////////////////// SOCKET SocketPrototype::openSocket(bool blocking) { SOCKET sockfd = SOCK_MAX; try { sockfd = socket(serv_addr_.sa_family, SOCK_STREAM, 0); if (sockfd == SOCK_MAX) { - throw SocketError("failed to create socket"); + throw SocketError(std::format( + "failed to create socket for {}", name_)); } auto result = connect(sockfd, &serv_addr_, sizeof(serv_addr_)); if (result < 0) { closeSocket(sockfd); - throw SocketError("failed to connect to server"); + throw SocketError(std::format( + "failed to connect to server for {}", name_)); } setBlocking(sockfd, blocking); } catch (const SocketError &) { @@ -128,7 +118,6 @@ SOCKET SocketPrototype::openSocket(bool blocking) return sockfd; } -/////////////////////////////////////////////////////////////////////////////// void SocketPrototype::closeSocket(SOCKET& sockfd) { if (sockfd == SOCK_MAX) { @@ -142,7 +131,7 @@ void SocketPrototype::closeSocket(SOCKET& sockfd) sockfd = SOCK_MAX; } -/////////////////////////////////////////////////////////////////////////////// +//////// bool SocketPrototype::testConnection() { try { @@ -157,7 +146,7 @@ bool SocketPrototype::testConnection() } } -//////////////////////////////////////////////////////////////////////////////// +//////// void SocketPrototype::setBlocking(SOCKET sock, bool setblocking) { if (sock == SOCK_MAX) { @@ -166,7 +155,8 @@ void SocketPrototype::setBlocking(SOCKET sock, bool setblocking) #ifdef WIN32 unsigned long mode = (unsigned long)!setblocking; if (ioctlsocket(sock, FIONBIO, &mode) != 0) { - throw SocketError("failed to set blocking mode on socket"); + throw SocketError(std::format( + "failed to set blocking mode on socket {}", name_)); } #else int flags = fcntl(sock, F_GETFL, 0); @@ -177,41 +167,43 @@ void SocketPrototype::setBlocking(SOCKET sock, bool setblocking) flags = setblocking ? (flags & ~O_NONBLOCK) : (flags | O_NONBLOCK); int rt = fcntl(sock, F_SETFL, flags); if (rt != 0) { - std::cout << "fcntl returned " << rt << std::endl; - std::cout << "error: " << strerror(errno); - throw SocketError("failed to set blocking mode on socket"); + throw SocketError(std::format( + "failed to set blocking mode on socket {}", name_)); } #endif blocking_ = setblocking; } -/////////////////////////////////////////////////////////////////////////////// +bool SocketPrototype::isBlocking() const +{ + return blocking_; +} + +//////// void SocketPrototype::listen(AcceptCallback callback, SOCKET& sockfd) { try { sockfd = socket(serv_addr_.sa_family, SOCK_STREAM, 0); if (sockfd == SOCK_MAX) { - throw SocketError("failed to create socket"); + throw SocketError(std::format("failed to create socket {}", name_)); } if (::bind(sockfd, &serv_addr_, sizeof(serv_addr_)) < 0) { closeSocket(sockfd); - throw SocketError("failed to bind socket"); + throw SocketError(std::format("failed to bind socket {}", name_)); } if (::listen(sockfd, 10) < 0) { closeSocket(sockfd); - throw SocketError("failed to listen to socket"); + throw SocketError(std::format("failed to listen to socket {}", name_)); } } catch (const SocketError&) { closeSocket(sockfd); return; } - std::stringstream errorss; std::exception_ptr exceptptr = nullptr; - struct pollfd pfd; pfd.fd = sockfd; pfd.events = POLLIN; @@ -233,13 +225,15 @@ void SocketPrototype::listen(AcceptCallback callback, SOCKET& sockfd) #else auto errornum = errno; #endif - errorss << "poll() error in readFromSocketThread: " << errornum; - LOGERR << errorss.str(); - throw SocketError(errorss.str()); + auto errStr = std::format( + "poll error during listen ({}): {}", + name_, errornum); + LOGERR << errStr; + throw SocketError(errStr); } if (pfd.revents & POLLNVAL) { - throw SocketError("POLLNVAL in readFromSocketThread"); + throw SocketError(std::format("POLLNVAL in listen ({})", name_)); } //exceptions @@ -247,9 +241,9 @@ void SocketPrototype::listen(AcceptCallback callback, SOCKET& sockfd) //TODO: grab socket error code, pass error to callback //break out of poll loop - errorss << "POLLERR error in readFromSocketThread"; - LOGERR << errorss.str(); - throw SocketError(errorss.str()); + auto errStr = std::format("POLLERR error in listen ({})", name_); + LOGERR << errStr; + throw SocketError(errStr); } if (pfd.revents & POLLIN) { @@ -272,24 +266,19 @@ void SocketPrototype::listen(AcceptCallback callback, SOCKET& sockfd) //// PersistentSocket // /////////////////////////////////////////////////////////////////////////////// -PersistentSocket::PersistentSocket(const std::string& addr, port_t port) : - SocketPrototype(addr, port) +PersistentSocket::PersistentSocket(const std::string& addr, port_t port, + const std::string& name) : + SocketPrototype(addr, port, name) { - shutdownProm_ = std::make_unique>(); - shutdownFut_ = shutdownProm_->get_future(); init(); } -//////// -PersistentSocket::PersistentSocket(SOCKET sockfd) : - SocketPrototype(), sockfd_(sockfd) +PersistentSocket::PersistentSocket(SOCKET sockfd, const std::string& name) : + SocketPrototype(name), sockfd_(sockfd) { - shutdownProm_ = std::make_unique>(); - shutdownFut_ = shutdownProm_->get_future(); init(); } -//////// PersistentSocket::~PersistentSocket() { for (auto& thr : threads_) { @@ -309,12 +298,16 @@ bool PersistentSocket::isValid() const return sockfd_ != SOCK_MAX; } -//////// bool PersistentSocket::testConnection() { return isValid(); } +bool PersistentSocket::running() const +{ + return run_.load(std::memory_order_relaxed); +} + /////////////////////////////////////////////////////////////////////////////// #ifndef _WIN32 void PersistentSocket::socketService_nix() @@ -371,7 +364,9 @@ void PersistentSocket::socketService_nix() } else if (status == -1) { //poll error, process and exit loop auto errornum = errno; - LOGERR << "poll() error in socketService_nix: " << errornum; + LOGERR << std::format( + "poll error in socket service ({}): {}", + name_, errornum); break; } @@ -389,13 +384,13 @@ void PersistentSocket::socketService_nix() } if (pfd[1].revents & POLLNVAL) { - LOGERR << "POLLNVAL in socketService_nix"; + LOGERR << std::format("POLLNVAL in socket service ({})", name_); } //exceptions if (pfd[1].revents & POLLERR) { //break out of poll loop - LOGERR << "POLLERR error in socketService_nix"; + LOGERR << std::format("POLLERR error in socket service ({})", name_); break; } @@ -406,7 +401,7 @@ void PersistentSocket::socketService_nix() if (readAmt == 0) { //pollin notified socket is ready to read but buffer is empty //treat as socket has been cleaned up - LOGDEBUG << "socket cleaned up"; + LOGDEBUG << std::format("socket \"{}\" cleaned up", name_); break; } //read it and push it into the queue @@ -417,11 +412,15 @@ void PersistentSocket::socketService_nix() readQueue_.push_back(std::move(readdata)); } else { LOGWARN << std::format( - "failed to read socket data with error: {}! dropping socket", errno); + "failed to read data from socket {} with error: {}!", + name_, errno); break; } } else { - LOGWARN << "ioctl failed with error: " << errno; + LOGWARN << std::format( + "ioctl for {} failed with error: {}", + name_, errno); + break; } } @@ -494,7 +493,7 @@ void PersistentSocket::socketService_win() } if (ev == WSA_WAIT_FAILED) { - LOGERR << "WSAWaitForMultipleEvents failed"; + LOGERR << std::format("WSAWaitForMultipleEvents failed ({})", name_); break; } @@ -512,7 +511,7 @@ void PersistentSocket::socketService_win() WSANETWORKEVENTS networkevents; if (WSAEnumNetworkEvents(sockfd_, events_[0], &networkevents) == SOCKET_ERROR) { - LOGERR << "error getting network events for socket"; + LOGERR << std::format("error getting network events for {}", name_); break; } @@ -532,7 +531,7 @@ void PersistentSocket::socketService_win() if (errornum == EAGAIN || errornum == EWOULDBLOCK) { break; } - LOGERR << "error reading socket, aborting"; + LOGERR << std::format("error reading socket {}, aborting", name_); break; } @@ -554,7 +553,8 @@ void PersistentSocket::socketService_win() } if (networkevents.lNetworkEvents & FD_CLOSE) { - LOGERR << "socket was closed: " << int(networkevents.iErrorCode[FD_CLOSE_BIT]); + LOGERR << std::format("socket {} was closed with code: {}", + name_, (int)networkevents.iErrorCode[FD_CLOSE_BIT]); break; } } @@ -564,7 +564,7 @@ void PersistentSocket::socketService_win() } #endif -/////////////////////////////////////////////////////////////////////////////// +//////// void PersistentSocket::queuePayloadForWrite(std::vector& payload) { if (payload.empty()) { @@ -578,7 +578,7 @@ void PersistentSocket::queuePayloadForWrite(std::vector& payload) signalService(0); } -/////////////////////////////////////////////////////////////////////////////// +//////// void PersistentSocket::readService() { while (true) { @@ -596,7 +596,6 @@ void PersistentSocket::readService() respond(empty); } -/////////////////////////////////////////////////////////////////////////////// bool PersistentSocket::processPacket( std::vector& packet, std::vector& payload) { @@ -604,7 +603,7 @@ bool PersistentSocket::processPacket( return true; } -/////////////////////////////////////////////////////////////////////////////// +//////// void PersistentSocket::signalService(uint8_t signal) { //0 to trigger a pollout, 1 to exit poll loop @@ -621,9 +620,10 @@ void PersistentSocket::signalService(uint8_t signal) #endif } -/////////////////////////////////////////////////////////////////////////////// +//////// void PersistentSocket::init() { + shutdownFut_ = shutdownProm_.get_future(); run_.store(false, std::memory_order_relaxed); #ifndef _WIN32 @@ -633,7 +633,6 @@ void PersistentSocket::init() #endif } -/////////////////////////////////////////////////////////////////////////////// void PersistentSocket::initPipes() { cleanUpPipes(); @@ -647,7 +646,6 @@ void PersistentSocket::initPipes() #endif } -/////////////////////////////////////////////////////////////////////////////// void PersistentSocket::cleanUpPipes() { for (unsigned i = 0; i < 2; i++) { @@ -665,7 +663,7 @@ void PersistentSocket::cleanUpPipes() } } -/////////////////////////////////////////////////////////////////////////////// +//////// bool PersistentSocket::openSocket(bool blocking) { if (!addr_.empty() && port_ != 0 && port_ != UINT16_MAX && @@ -675,7 +673,6 @@ bool PersistentSocket::openSocket(bool blocking) return isValid(); } -/////////////////////////////////////////////////////////////////////////////// int PersistentSocket::getSocketName(struct sockaddr& sa) { #ifdef _WIN32 @@ -686,7 +683,6 @@ int PersistentSocket::getSocketName(struct sockaddr& sa) return getsockname(sockfd_, &sa, &namelen); } -/////////////////////////////////////////////////////////////////////////////// int PersistentSocket::getPeerName(struct sockaddr& sa) { #ifdef _WIN32 @@ -697,7 +693,6 @@ int PersistentSocket::getPeerName(struct sockaddr& sa) return getpeername(sockfd_, &sa, &namelen); } -/////////////////////////////////////////////////////////////////////////////// bool PersistentSocket::connectToRemote() { if (run_.load(std::memory_order_relaxed)) { @@ -723,7 +718,9 @@ bool PersistentSocket::connectToRemote() try { this->socketService(); } catch (const SocketError&) { - LOGERR << "error in socket service, shutting down connection"; + LOGERR << std::format( + "error in socket service for {}, shutting down connection", + name_); shutdown(); } }; @@ -733,16 +730,17 @@ bool PersistentSocket::connectToRemote() return true; } -/////////////////////////////////////////////////////////////////////////////// +//////// void PersistentSocket::shutdown() { std::unique_lock lock(shutdownMutex_); - if (shutdownFut_.wait_for(0s) == std::future_status::ready) { + auto futCopy = shutdownFut_; + if (futCopy.wait_for(0s) == std::future_status::ready) { return; } readQueue_.terminate(); signalService(1); - shutdownProm_->set_value(true); + shutdownProm_.set_value(); } void PersistentSocket::blockUntilClosed() const @@ -756,13 +754,14 @@ void PersistentSocket::blockUntilClosed() const //// SimpleSocket // /////////////////////////////////////////////////////////////////////////////// -SimpleSocket::SimpleSocket(const std::string& addr, port_t port) : - SocketPrototype(addr, port) +SimpleSocket::SimpleSocket(const std::string& addr, port_t port, + const std::string& name) : + SocketPrototype(addr, port, name) {} //////// -SimpleSocket::SimpleSocket(SOCKET sockfd) : - SocketPrototype(), sockfd_(sockfd) +SimpleSocket::SimpleSocket(SOCKET sockfd, const std::string& name) : + SocketPrototype(name), sockfd_(sockfd) {} //////// @@ -771,19 +770,25 @@ SimpleSocket::~SimpleSocket() closeSocket(sockfd_); } +void SimpleSocket::shutdown() +{ + closeSocket(sockfd_); +} + //////// SocketType SimpleSocket::type() const { return SocketType::Simple; } -//////// -SOCKET SimpleSocket::getSockFD() const +bool SimpleSocket::running() const { - return sockfd_; + //this is meant for REST with keep-alive behavior, + //return true so long as the socket is valid + return sockfd_ != SOCK_MAX; } -/////////////////////////////////////////////////////////////////////////////// +//////// void SimpleSocket::pushPayload( std::unique_ptr write_payload, std::shared_ptr read_payload) @@ -807,20 +812,13 @@ void SimpleSocket::pushPayload( read_payload->callbackReturn_->callback(bdr); } -/////////////////////////////////////////////////////////////////////////////// -void SimpleSocket::listen(AcceptCallback acb) -{ - SocketPrototype::listen(std::move(acb), sockfd_); -} - -/////////////////////////////////////////////////////////////////////////////// -void SimpleSocket::shutdown() +int SimpleSocket::writeToSocket(std::vector& payload) { - closeSocket(sockfd_); + return send(sockfd_, (char*)&payload[0], payload.size(), 0); } -/////////////////////////////////////////////////////////////////////////////// -std::vector SimpleSocket::readFromSocket(void) +//////// +std::vector SimpleSocket::readFromSocket() { //exit after one read size_t readIncrement = 8192; @@ -846,19 +844,21 @@ std::vector SimpleSocket::readFromSocket(void) #else auto errornum = errno; #endif - LOGERR << "poll() error in readFromSocketThread: " << errornum; + LOGERR << std::format( + "poll error in read service ({}): {}", + name_, errornum); break; } if (pfd.revents & POLLNVAL) { - LOGERR << "POLLNVAL in readFromSocketThread"; + LOGERR << std::format("POLLNVAL in read service ({})", name_); break; } //exceptions if (pfd.revents & POLLERR) { //break out of poll loop - LOGERR << "POLLERR error in readFromSocketThread"; + LOGERR << std::format("POLLERR in read service ({})", name_); break; } @@ -883,7 +883,7 @@ std::vector SimpleSocket::readFromSocket(void) break; } #endif - LOGERR << "recv error: " << errornum; + LOGERR << std::format("recv error for {}: {}", name_, errornum); break; } @@ -895,7 +895,7 @@ std::vector SimpleSocket::readFromSocket(void) } if (readAmt == 0) { - LOGINFO << "POLLIN recv return 0"; + LOGINFO << std::format("recv return 0 for {}", name_); break; } @@ -913,13 +913,7 @@ std::vector SimpleSocket::readFromSocket(void) return {}; } -/////////////////////////////////////////////////////////////////////////////// -int SimpleSocket::writeToSocket(std::vector& payload) -{ - return send(sockfd_, (char*)&payload[0], payload.size(), 0); -} - -/////////////////////////////////////////////////////////////////////////////// +//////// bool SimpleSocket::connectToRemote() { if (sockfd_ == SOCK_MAX) { @@ -928,20 +922,31 @@ bool SimpleSocket::connectToRemote() return sockfd_ != SOCK_MAX; } +void SimpleSocket::listen(AcceptCallback acb) +{ + SocketPrototype::listen(std::move(acb), sockfd_); +} + bool SimpleSocket::checkSocket(const std::string& ip, port_t port) { - SimpleSocket testSock(ip, port); + SimpleSocket testSock(ip, port, {}); return testSock.testConnection(); } +SOCKET SimpleSocket::getSockFD() const +{ + return sockfd_; +} + /////////////////////////////////////////////////////////////////////////////// // //// ListenServer // /////////////////////////////////////////////////////////////////////////////// -ListenServer::ListenServer(const std::string& addr, port_t port) +ListenServer::ListenServer(const std::string& addr, port_t port, + const std::string& name) { - listenSocket_ = std::make_unique(addr, port); + listenSocket_ = std::make_unique(addr, port, name); listenSocket_->verbose_ = false; } @@ -1011,7 +1016,8 @@ void ListenServer::acceptProcess(AcceptStruct aStruct) auto ss = std::make_unique(); //create BinarySocket object from sockfd - ss->sock_ = std::make_shared(aStruct.sockfd_); + ss->sock_ = std::make_shared( + aStruct.sockfd_, std::string{aStruct.saddr_.sa_data}); ss->sock_->verbose_ = false; //start read lambda thread @@ -1156,4 +1162,3 @@ bool Socket_WritePayload::isSingleSegment() const AcceptStruct::AcceptStruct() : addrlen_(sizeof(saddr_)) {} - diff --git a/cppForSwig/Network/SocketObject.h b/cppForSwig/Network/SocketObject.h index 49c1ae128..7c534ac15 100644 --- a/cppForSwig/Network/SocketObject.h +++ b/cppForSwig/Network/SocketObject.h @@ -94,31 +94,29 @@ namespace Armory private: bool blocking_ = true; - protected: - public: - typedef std::function&)> SequentialReadCallback; typedef std::function AcceptCallback; protected: - const size_t maxread_ = 4*1024*1024; - struct sockaddr serv_addr_; const std::string addr_; const port_t port_; + const std::string name_; bool verbose_ = true; private: void init(void); protected: - SocketPrototype(void); + SocketPrototype(const std::string&); void setBlocking(SOCKET, bool); void listen(AcceptCallback, SOCKET&); public: - SocketPrototype(const std::string&, port_t, bool = true); + SocketPrototype( + const std::string&, port_t, + const std::string&, bool = true); virtual ~SocketPrototype(void) = 0; virtual bool testConnection(void); @@ -135,7 +133,7 @@ namespace Armory const std::string& getAddrStr(void) const; //override me - virtual bool running(void) const; + virtual bool running(void) const = 0; }; class SimpleSocket : public SocketPrototype @@ -147,21 +145,24 @@ namespace Armory int writeToSocket(std::vector&); public: - SimpleSocket(const std::string&, port_t); - SimpleSocket(SOCKET); + SimpleSocket(const std::string&, port_t, const std::string&); + SimpleSocket(SOCKET, const std::string&); ~SimpleSocket(void); - SocketType type(void) const override; SOCKET getSockFD(void) const; - void pushPayload( - std::unique_ptr, - std::shared_ptr) override; std::vector readFromSocket(void); void shutdown(void); void listen(AcceptCallback); + + //overrides bool connectToRemote(void) override; + SocketType type(void) const override; + bool running(void) const override; + void pushPayload( + std::unique_ptr, + std::shared_ptr) override; - // + //statics static bool checkSocket(const std::string&, port_t); }; @@ -177,8 +178,8 @@ namespace Armory size_t writeOffset_ = 0; std::atomic run_; - std::shared_future shutdownFut_; - std::unique_ptr> shutdownProm_; + std::shared_future shutdownFut_; + std::promise shutdownProm_; std::mutex shutdownMutex_; #ifdef _WIN32 @@ -208,18 +209,21 @@ namespace Armory void queuePayloadForWrite(std::vector&); public: - PersistentSocket(const std::string&, port_t); - PersistentSocket(SOCKET); + PersistentSocket(const std::string&, port_t, const std::string&); + PersistentSocket(SOCKET, const std::string&); ~PersistentSocket(void); void shutdown(void); bool openSocket(bool); int getSocketName(struct sockaddr& ); int getPeerName(struct sockaddr&); - bool connectToRemote(void) override; bool isValid(void) const; bool testConnection(void); void blockUntilClosed(void) const; + + //overrides + bool connectToRemote(void) override; + bool running(void) const override; }; //////// @@ -253,7 +257,7 @@ namespace Armory ListenServer(const ListenServer&) = delete; public: - ListenServer(const std::string&, port_t); + ListenServer(const std::string&, port_t, const std::string&); ~ListenServer(void); void start(ReadCallback); diff --git a/cppForSwig/Network/StringSockets.cpp b/cppForSwig/Network/StringSockets.cpp index f354e7972..9fa755a75 100644 --- a/cppForSwig/Network/StringSockets.cpp +++ b/cppForSwig/Network/StringSockets.cpp @@ -15,8 +15,9 @@ using namespace Armory::Network; /////////////////////////////////////////////////////////////////////////////// // HttpSocket -HttpSocket::HttpSocket(const std::string& addr, port_t port) : - SimpleSocket(addr, port) +HttpSocket::HttpSocket(const std::string& addr, port_t port, + const std::string& name) : + SimpleSocket(addr, port, name) { messageWithPrecacheHeaders_ = make_unique(getAddrStr()); } @@ -80,7 +81,8 @@ bool HttpSocket::processPacket( //no content-length header was found, abort if (currentRead_.content_length_ == -1) { - throw HttpError("failed to find http header response packet"); + throw HttpError(std::format( + "failed to find http header response packet ({})", name_)); } //check the total amount of data accumulated matches the advertised diff --git a/cppForSwig/Network/StringSockets.h b/cppForSwig/Network/StringSockets.h index a4cf4f4ba..08c908dfb 100644 --- a/cppForSwig/Network/StringSockets.h +++ b/cppForSwig/Network/StringSockets.h @@ -90,7 +90,7 @@ namespace Armory std::string getHttpPayload(const char*, size_t); public: - HttpSocket(const std::string&, port_t); + HttpSocket(const std::string&, port_t, const std::string&); static size_t getHttpBodyOffset(const char*, size_t); SocketType type(void) const override; diff --git a/cppForSwig/Network/WebSocketClient.cpp b/cppForSwig/Network/WebSocketClient.cpp index dcfc1b227..1a70655a3 100755 --- a/cppForSwig/Network/WebSocketClient.cpp +++ b/cppForSwig/Network/WebSocketClient.cpp @@ -40,8 +40,8 @@ static struct lws_protocols protocols[] = { WebSocketClient::WebSocketClient(const std::string& addr, port_t port, std::shared_ptr peers, bool oneWayAuth, std::shared_ptr cbPtr) : - SocketPrototype(addr, port, false), - servName_(std::format("{}:{}", addr_, port_)), + SocketPrototype(addr, port, std::format("wsclient_{}", addr), false), + servName_(std::format("{}:{}", addr, port)), callbackPtr_(cbPtr), peerStore_(peers) { count_.store(0, std::memory_order_relaxed); @@ -182,8 +182,9 @@ struct lws_context* WebSocketClient::init() //info.ws_ping_pong_interval = 60; auto contextptr = lws_create_context(&info); - if (contextptr == NULL) { - throw LWS_Error("failed to create LWS context"); + if (contextptr == nullptr) { + throw LWS_Error(std::format( + "failed to create LWS context for {}", name_)); } //connect to server @@ -194,8 +195,9 @@ struct lws_context* WebSocketClient::init() const char *prot, *p; char path[300]; if (lws_parse_uri((char*)addr_.c_str(), &prot, &i.address, &i.port, &p) != 0) { - LOGERR << "failed to parse server URI"; - throw LWS_Error("failed to parse server URI"); + auto errStr = std::format("failed to parse URI for {}", name_); + LOGERR << errStr; + throw LWS_Error(errStr); } path[0] = '/'; @@ -277,10 +279,12 @@ void WebSocketClient::cleanup() if (readThr_.joinable()) { readThr_.join(); } - } catch(const std::system_error& e) { - LOGERR << "failed to join on client threads with error:"; - LOGERR << e.what(); - throw e; + } catch (const std::system_error& e) { + auto errStr = std::format( + "error while joining threads during {} cleaning up: {}", + name_, e.what()); + LOGERR << errStr; + throw errStr; } readPackets_.clear(); @@ -290,7 +294,7 @@ void WebSocketClient::cleanup() auto notifs = payload.initNotifs(1); auto error = notifs[0].initError(); error.setCode(-1); - error.setErrStr("LWS client disconnected"); + error.setErrStr(std::format("{} disconnected", name_)); auto flat = capnp::messageToFlatArray(message); auto bytes = flat.asBytes(); @@ -331,7 +335,7 @@ void WebSocketClient::cleanup() thr.join(); } } - LOGINFO << "lws client cleaned up"; + LOGINFO << std::format("{} was cleaned up", name_); } //////// @@ -352,12 +356,18 @@ int WebSocketClient::lwsServiceHandler(struct lws* wsi, case LWS_CALLBACK_CLIENT_CONNECTION_ERROR: { - LOGERR << "lws client connection error"; + if (instance != nullptr) { + LOGERR << std::format("connection error in {}", + instance->name_); + } else { + LOGERR << "LWS connection error in unknown instance!"; + } + if (len > 0) { auto errstr = (char*)in; - LOGERR << " error message: " << errstr; + LOGERR << " . error message: " << errstr; } else { - LOGERR << "no error message was provided by lws"; + LOGERR << "no error message was provided by LWS"; } [[fallthrough]]; } @@ -403,21 +413,20 @@ int WebSocketClient::lwsServiceHandler(struct lws* wsi, auto packet = instance->currentWriteMessage_.consumeNextPacket(); auto body = (uint8_t*)packet.getPtr() + LWS_PRE; - auto m = lws_write(wsi, + auto bytesWritten = lws_write(wsi, body, packet.getSize() - LWS_PRE, LWS_WRITE_BINARY); - if (m != (int)packet.getSize() - (int)LWS_PRE) { - LOGERR << "failed to send packet of size"; - LOGERR << "packet is " << packet.getSize() << - " bytes, sent " << m << " bytes"; + if (bytesWritten != (int)packet.getSize() - (int)LWS_PRE) { + LOGERR << std::format( + "{} failed to write packet of size {} bytes, sent {} instead", + instance->name_, packet.getSize() - LWS_PRE, bytesWritten); } if (instance->currentWriteMessage_.isDone()) { instance->currentWriteMessage_.clear(); instance->count_.fetch_add(1, std::memory_order_relaxed); } - break; } @@ -484,7 +493,8 @@ void WebSocketClient::readService() } if (bip151Connection_->getBIP150State() != BIP150State::SUCCESS) { - LOGWARN << "encryption layer is uninitialized, aborting connection"; + LOGWARN << std::format( + "AEAD for {} is in invalid state, dropping socket", name_); shutdown(); return; } @@ -521,7 +531,8 @@ void WebSocketClient::readService() readPackets_.erase(msgid); currentReadMessage_.reset(); } else { - LOGWARN << "there is no callback registered for msg id: " << msgid; + LOGWARN << std::format( + "{} has no callback registered for msg id {}", name_, msgid); currentReadMessage_.reset(); } } @@ -559,8 +570,9 @@ bool WebSocketClient::processAEADHandshake(const WebSocketMessagePartial& msgObj /*packet is server's pubkey, do we have it?*/ if (!bip151Connection_->isOneWayAuth()) { - LOGERR << "Trying to connect to 1-way server as a 2-way client." << - " Aborting!"; + LOGERR << std::format( + "{} is trying to connect to 1-way server as a 2-way client!", + name_); return false; } @@ -575,8 +587,8 @@ bool WebSocketClient::processAEADHandshake(const WebSocketMessagePartial& msgObj case ArmoryAEAD::BIP151_PayloadType::EncInit: { if (bip151Connection_->isOneWayAuth() && !serverPubkeyAnnounce_) { - LOGERR << "trying to connect to 2-way server as 1-way client." << - " Aborting!"; + LOGERR << std::format( + "{} is trying to connect to 2-way server as 1-way client!", name_); return false; } break; diff --git a/cppForSwig/Node/BitcoinP2P.cpp b/cppForSwig/Node/BitcoinP2P.cpp index 96530c2b2..5799b41c4 100644 --- a/cppForSwig/Node/BitcoinP2P.cpp +++ b/cppForSwig/Node/BitcoinP2P.cpp @@ -1280,9 +1280,9 @@ void P2P::Iface::requestTx(InvVector invVec) //////////////////////////////////////////////////////////////////////////////// // Peer P2P::Peer::Peer( - const std::string& addrV4, Network::port_t port, + const std::string& addrV4, Network::port_t port, const std::string& name, uint32_t magicword, bool watcher) : - Iface(magicword, watcher), addr_(addrV4), port_(port) + Iface(magicword, watcher), addr_(addrV4), port_(port), name_{name} { init(); } @@ -1304,15 +1304,20 @@ void P2P::Peer::init() void P2P::Peer::connectToNode(bool async) { std::unique_lock lock(connectMutex_, std::defer_lock); - if (!lock.try_lock() || connectedPromise_ != nullptr) { + if (!lock.try_lock()) { //return if another thread is already here - throw Network::SocketError("another connect attempt is underway"); + throw Network::SocketError(std::format( + "another connect attempt is underway for {}", name_)); } - connectedPromise_ = std::unique_ptr>(new std::promise()); - auto connectedFuture = connectedPromise_->get_future(); + std::promise connectedPromise; + auto connectedFuture = connectedPromise.get_future(); - std::thread connectthread([this]{ connectLoop(); }); + std::thread connectthread( + [this](std::promise prom) + { connectLoop(std::move(prom)); }, + std::move(connectedPromise) + ); if (connectthread.joinable()) { connectthread.detach(); } @@ -1321,16 +1326,16 @@ void P2P::Peer::connectToNode(bool async) return; } - connectedFuture.get(); + connectedFuture.wait(); if (process_except_ != nullptr) { std::rethrow_exception(process_except_); } } -void P2P::Peer::connectLoop() +void P2P::Peer::connectLoop(std::promise connectedPromise) { auto waitBeforeReconnect = 0ms; - std::promise shutdownPromise; + std::promise shutdownPromise; shutdownFuture_ = shutdownPromise.get_future(); if (!invTxLambda_) { @@ -1341,7 +1346,7 @@ void P2P::Peer::connectLoop() //setup fresh connection dataStack_ = std::make_shared< Threading::BlockingQueue>>(); - socket_ = std::make_unique(addr_, port_, dataStack_); + socket_ = std::make_unique(addr_, port_, name_, dataStack_); verackPromise_ = std::make_unique>(); auto verackFuture = verackPromise_->get_future(); @@ -1377,28 +1382,32 @@ void P2P::Peer::connectLoop() try { //send version if (socket_->getSocketName(clientsocketaddr) != 0) { - throw Network::SocketError("failed to get client sockaddr"); + throw Network::SocketError( + std::format("{} failed to get client sockaddr", name_)); } if (socket_->getPeerName(node_addr_) != 0) { - throw Network::SocketError("failed to get peer sockaddr"); + throw Network::SocketError( + std::format("failed to get peer sockaddr", name_)); } // Services, for future extensibility uint32_t services = NODE_WITNESS; version->setVersionHeaderIPv4(70012, services, timestamp, node_addr_, clientsocketaddr); - version->userAgent_ = "Armory:0.96.5"; + version->userAgent_ = "Armory:0.96.99"; version->startHeight_ = -1; sendMessage(std::move(version)); //wait on verack verackFuture.get(); verackPromise_.reset(); - LOGINFO << "Connected to Bitcoin node"; + LOGINFO << std::format("{} is connected to Bitcoin node", name_); updateNodeStatus(true); - //signal calling thread - connectedPromise_->set_value(true); + try { + //notify calling thread the connection is ready on first success + connectedPromise.set_value(); + } catch (const std::future_error&) {} waitBeforeReconnect = 0ms; //signal new blocks for good measure @@ -1416,10 +1425,10 @@ void P2P::Peer::connectLoop() if (socket_->isValid()) { socket_->shutdown(); } - LOGINFO << "Disconnected from Bitcoin node"; + LOGINFO << std::format("{} disconnected from Bitcoin node", name_); updateNodeStatus(false); } - shutdownPromise.set_value(true); + shutdownPromise.set_value(); } //////// @@ -1657,7 +1666,9 @@ void P2P::Peer::shutdown() if (socket_ != nullptr) { socket_->shutdown(); - shutdownFuture_.wait(); + try { + shutdownFuture_.wait(); + } catch (const std::future_error&) {} } //have to call the parent class shutdown explicitly @@ -1679,9 +1690,10 @@ bool P2P::Peer::connected() const //////////////////////////////////////////////////////////////////////////////// // Socket P2P::Socket::Socket( - const std::string& addr, Network::port_t port, + const std::string& addr, Network::port_t port, const std::string& name, std::shared_ptr>> readStack) : - PersistentSocket(addr, port), readDataStack_(readStack) + PersistentSocket(addr, port, name), + readDataStack_(readStack) {} SocketType P2P::Socket::type() const diff --git a/cppForSwig/Node/BitcoinP2P.h b/cppForSwig/Node/BitcoinP2P.h index d3948865c..7483cb546 100644 --- a/cppForSwig/Node/BitcoinP2P.h +++ b/cppForSwig/Node/BitcoinP2P.h @@ -371,7 +371,7 @@ namespace Node std::vector>> readDataStack_; public: - Socket(const std::string&, Armory::Network::port_t, + Socket(const std::string&, Armory::Network::port_t, const std::string&, std::shared_ptr>>); SocketType type(void) const override; @@ -443,12 +443,13 @@ namespace Node { private: const std::string addr_; - Armory::Network::port_t port_; + const Armory::Network::port_t port_; + const std::string name_; + struct sockaddr node_addr_; std::unique_ptr socket_; std::mutex connectMutex_, pollMutex_, writeMutex_; - std::unique_ptr> connectedPromise_ = nullptr; std::unique_ptr> verackPromise_ = nullptr; std::atomic nodeConnected_; @@ -458,12 +459,12 @@ namespace Node std::exception_ptr select_except_ = nullptr; std::exception_ptr process_except_ = nullptr; - std::future shutdownFuture_; + std::future shutdownFuture_; uint32_t topBlock_ = UINT32_MAX; private: void init(void); - void connectLoop(void); + void connectLoop(std::promise); void processDataStackThread(void); void processPayload(std::vector>); @@ -484,7 +485,7 @@ namespace Node public: Peer(const std::string&, Armory::Network::port_t, - MagicWordType, bool); + const std::string&, MagicWordType, bool); ~Peer(void); //virtuals diff --git a/cppForSwig/Node/nodeRPC.cpp b/cppForSwig/Node/nodeRPC.cpp index 8dee0b715..3ffe82ddb 100644 --- a/cppForSwig/Node/nodeRPC.cpp +++ b/cppForSwig/Node/nodeRPC.cpp @@ -374,7 +374,9 @@ RPC::FeeEstimateResult RPC::Client::getFeeByte( void RPC::Client::aggregateFeeEstimates() { //get fee/byte for 2-3-4-5-6-10-20 confs on both strategies - Network::HttpSocket sock("127.0.0.1", Config::NetworkSettings::rpcPort()); + Network::HttpSocket sock( + "127.0.0.1", Config::NetworkSettings::rpcPort(), + "satoshiRPC_fees"); if (!setupConnection(sock)) { throw RpcError("aggregateFeeEstimates: failed to setup RPC socket"); } @@ -576,9 +578,11 @@ bool RPC::Client::shutdown() //////// std::string RPC::Client::queryRPC(JSON::Object& request) { - Network::HttpSocket sock("127.0.0.1", Config::NetworkSettings::rpcPort()); + Network::HttpSocket sock( + "127.0.0.1", Config::NetworkSettings::rpcPort(), + "satoshiRPC"); if (!setupConnection(sock)) { - throw RpcError("node_down"); + throw RpcError("satoshi_rpc_down"); } return queryRPC(sock, request); } diff --git a/cppForSwig/Utils/ArmoryConfig.cpp b/cppForSwig/Utils/ArmoryConfig.cpp index e6fb3d6b7..ba3803c06 100755 --- a/cppForSwig/Utils/ArmoryConfig.cpp +++ b/cppForSwig/Utils/ArmoryConfig.cpp @@ -886,12 +886,12 @@ void NetworkSettings::createNodes() auto magicBytes = BitcoinSettings::getMagicBytes(); if (DBSettings::getServiceType() == SERVICE_WEBSOCKET) { bitcoinNodes_.first = std::make_shared( - "127.0.0.1", btcPort_, + "127.0.0.1", btcPort_, "satoshi_p2p_main", *(uint32_t*)magicBytes.getPtr(), false ); bitcoinNodes_.second = std::make_shared( - "127.0.0.1", btcPort_, + "127.0.0.1", btcPort_, "satoshi_p2p_watcher", *(uint32_t*)magicBytes.getPtr(), true ); diff --git a/cppForSwig/main.cpp b/cppForSwig/main.cpp index 734ad5d81..3aca7168f 100644 --- a/cppForSwig/main.cpp +++ b/cppForSwig/main.cpp @@ -60,7 +60,7 @@ int main(int argc, char* argv[]) auto logFilePath = Config::Pathing::logFilePath(LOG_FILE_NAME).string(); LOGDISABLESTDOUT(); - STARTLOGGING(logFilePath, LogLvlDebug); + STARTLOGGING(logFilePath, LogLvlInfo); if (Config::NetworkSettings::ephemeralPeers()) { if (Config::NetworkSettings::oneWayAuth()) { LOGERR << "--ephemeral and --oneWayAuth are mutually exclusive for db";