Implement silent payment indexing and scan.

include/bitcoin/database/impl/query/batch/silent.ipp

@@ -27,7 +27,17 @@ namespace libbitcoin {
 namespace database {
 
 TEMPLATE
-bool CLASS::scan_silent(const stopper& cancel, const ec_secret& scan_key,
+bool CLASS::scan_silent(const stopper& cancel,
+    const system::silent::batch::receiver& keys,
+    const silent_handler& callback) NOEXCEPT
+{
+    return scan_silent(cancel, keys, zero, store_.silent.count(), callback);
+}
+
+// Rows [first, last) must begin and end on transaction boundaries.
+TEMPLATE
+bool CLASS::scan_silent(const stopper& cancel,
+    const system::silent::batch::receiver& keys, size_t first, size_t last,
     const silent_handler& callback) NOEXCEPT
 {
     const auto prefix_ptr = store_.silent.prefix.get_memory();
@@ -43,24 +53,46 @@ bool CLASS::scan_silent(const stopper& cancel, const ec_secret& scan_key,
     const auto prefix = pointer_cast<prefix_t>(prefix_ptr.data());
     const auto compressed = pointer_cast<compressed_t>(compressed_ptr.data());
 
-    // Shortest column.
-    const auto count = store_.silent.count();
+    BC_ASSERT(first <= last && last <= store_.silent.count());
+    const auto count = last - first;
     const silent::batch batch
     {
-        .correlates = { correlate, count },
-        .prefixes = { prefix, count },
-        .points = { compressed, count }
+        .correlates = { std::next(correlate, first), count },
+        .prefixes = { std::next(prefix, first), count },
+        .points = { std::next(compressed, first), count }
     };
 
     // False return only implies canceled.
     // Callbacks invoked on caller thread if turbo is false.
-    silent::batch::scan(cancel, batch, scan_key, callback, store_.turbo());
+    silent::batch::scan(cancel, batch, keys, callback, store_.turbo());
     return !cancel;
 }
 
+// Rows are allocated zero-filled and a nonzero correlate publishes the row.
+TEMPLATE
+size_t CLASS::get_silent_frontier(size_t first) const NOEXCEPT
+{
+    using namespace system;
+    using word_t = table::silent_correlate::tx::integer;
+    const auto guard = store_.silent.guard();
+    const auto words = pointer_cast<word_t>(guard.data());
+    const auto count = store_.silent.count();
+
+    for (auto row = first; row < count; ++row)
+    {
+        std::atomic_ref<word_t> word{ *std::next(words, row) };
+        if (is_zero(word.load(std::memory_order_acquire)))
+            return row;
+    }
+
+    return count;
+}
+
 // setters
 // ----------------------------------------------------------------------------
 // Caller (node) controls which txs are indexed (e.g. by confirmed height).
+// The coinbase is the first tx archived for a block, so txs linked below it
+// were archived (and indexed) before it, as pooled or by another block.
 
 TEMPLATE
 bool CLASS::set_silent(const header_link& link, const block& block) NOEXCEPT
@@ -74,6 +106,8 @@ bool CLASS::set_silent(const header_link& link, const block& block) NOEXCEPT
     if (links.size() != count)
         return false;
 
+    const auto first = links.front();
+
     stopper fail{};
     std::vector<size_t> it(sub1(count));
     std::iota(it.begin(), it.end(), one);
@@ -87,7 +121,8 @@ bool CLASS::set_silent(const header_link& link, const block& block) NOEXCEPT
         if (fail.load(relaxed))
             return;
 
-        if (!set_silent(links.at(index), *txs->at(index)))
+        const auto& fk = links.at(index);
+        if (fk >= first && !set_silent(fk, *txs->at(index)))
             fail.store(true, relaxed);
     });
 
@@ -107,6 +142,8 @@ bool CLASS::set_silent(const header_link& link,
     if (links.size() != count)
         return false;
 
+    const auto first = links.front();
+
     stopper fail{};
     std::vector<size_t> it(sub1(count));
     std::iota(it.begin(), it.end(), one);
@@ -118,58 +155,61 @@ bool CLASS::set_silent(const header_link& link,
         if (fail.load(relaxed))
             return;
 
-        if (!set_silent(links.at(index), txs.at(index)))
+        const auto& fk = links.at(index);
+        if (fk >= first && !set_silent(fk, txs.at(index)))
             fail.store(true, relaxed);
     });
 
     return !fail.load(relaxed);
 }
 
+// Ineligible txs have no records.
 TEMPLATE
-bool CLASS::set_silent(const tx_link& link,
-    const transaction& BC_DEBUG_ONLY(tx)) NOEXCEPT
+bool CLASS::set_silent(const tx_link& link, const transaction& tx) NOEXCEPT
 {
-    BC_ASSERT(!tx.is_coinbase());
-    return set_silent_(link);
+    using namespace system::wallet;
+    ec_compressed summary{};
+    silent_payment::scan_outputs outputs{};
+    return !silent_payment::get_outputs(outputs, tx)
+        || !silent_payment::summarize(summary, tx)
+        || set_silent_(link, summary, outputs);
 }
 
 TEMPLATE
 bool CLASS::set_silent(const tx_link& link,
-    const transaction_view& BC_DEBUG_ONLY(tx)) NOEXCEPT
+    const transaction_view& tx) NOEXCEPT
 {
-    BC_ASSERT(!tx.is_coinbase());
-    return set_silent_(link);
+    using namespace system::wallet;
+    ec_compressed summary{};
+    silent_payment::scan_outputs outputs{};
+    return !silent_payment::get_outputs(outputs, tx)
+        || !silent_payment::summarize(summary, tx)
+        || set_silent_(link, summary, outputs);
 }
 
 // protected
 TEMPLATE
-bool CLASS::set_silent_(const tx_link& link) NOEXCEPT
+bool CLASS::set_silent_(const tx_link& link, const ec_compressed& summary,
+    const system::wallet::silent_payment::scan_outputs& outputs) NOEXCEPT
 {
     if (link.is_terminal())
         return false;
 
-    // Short-circuits with success on empty.
-    ////using namespace system::wallet;
-    ////silent_payment::scan_record record{};
-    ////if (!silent_payment::compute_scan_record(record, tx))
-    ////    return true;
-
-    // TODO: aliases for record above;
-    const ec_compressed key{};
-    const std::vector<uint64_t> prefixes{};
-
-    using correlate_t = table::silent_correlate::records;
-    using prefix_t = table::silent_prefix::put_ref;
+    // The prefix is ec_xonly[0..7] read as little-endian.
+    using namespace system;
+    using prefix_t = table::silent_prefix::integral;
+    std::vector<prefix_t> prefixes(outputs.size());
+    std::transform(outputs.cbegin(), outputs.cend(), prefixes.begin(),
+        [](const auto& output) NOEXCEPT
+        {
+            return unsafe_from_little_endian<prefix_t>(output.key.data());
+        });
+
+    using prefixes_t = table::silent_prefix::put_ref;
     using compressed_t = table::silent_compressed::put_ref;
 
-    // TODO: Caller must guard reads, this is writing into hot storage. This
-    // TODO: requires caller to chase writers and account for the last contig-
-    // TODO: uously populated row (for searching) and to update subscriptions
-    // TODO: with additional scans as this position increases.
     // ========================================================================
     const auto scope = get_transactor();
-
-    using namespace system;
     auto rows = possible_narrow_cast<silent_link::integer>(prefixes.size());
 
     // Allocate rows across all columns.
@@ -181,10 +221,23 @@ bool CLASS::set_silent_(const tx_link& link) NOEXCEPT
     const auto guard = store_.silent.guard();
 
     // Write values to each column in corresponding positions.
-    return
-        store_.silent.correlate.put(fk, correlate_t{ {}, rows, link }) &&
-        store_.silent.prefix.put(fk, prefix_t{ {}, prefixes }) &&
-        store_.silent.compressed.put(fk, compressed_t{ {}, rows, key });
+    if (!store_.silent.prefix.put(fk, prefixes_t{ {}, prefixes }) ||
+        !store_.silent.compressed.put(fk, compressed_t{ {}, rows, summary }))
+        return false;
+
+    // The guard is the correlate column, published last (get_silent_frontier).
+    using word_t = table::silent_correlate::tx::integer;
+    static_assert(schema::silent_correlate::minrow == sizeof(word_t));
+    const auto words = pointer_cast<word_t>(guard.data());
+    const auto value = native_to_little_end(link.value);
+
+    for (auto row = fk.value; row < fk.value + rows; ++row)
+    {
+        std::atomic_ref<word_t> word{ *std::next(words, row) };
+        word.store(value, std::memory_order_release);
+    }
+
+    return true;
     // ========================================================================
 }
 

include/bitcoin/database/impl/query/extent.ipp

@@ -282,6 +282,18 @@ bool CLASS::filter_enabled() const NOEXCEPT
     return store_.filter_bk.enabled() && store_.filter_tx.enabled();
 }
 
+TEMPLATE
+bool CLASS::silent_enabled() const NOEXCEPT
+{
+    return store_.get_envelope().silent_start_height != max_uint32;
+}
+
+TEMPLATE
+size_t CLASS::silent_start_height() const NOEXCEPT
+{
+    return store_.get_envelope().silent_start_height;
+}
+
 } // namespace database
 } // namespace libbitcoin
 

include/bitcoin/database/query.hpp

@@ -258,6 +258,8 @@ class query
     /// Optional/configured table state.
     bool address_enabled() const NOEXCEPT;
     bool filter_enabled() const NOEXCEPT;
+    bool silent_enabled() const NOEXCEPT;
+    size_t silent_start_height() const NOEXCEPT;
     size_t interval_span() const NOEXCEPT;
 
     /// Initialization (natural-keyed).
@@ -671,9 +673,16 @@ class query
     bool set_signatures(const system::chain::schnorr_signatures& sigs,
         const header_link& link, bool bank) NOEXCEPT;
 
-    /// Invoke callback for each candidate match, false implies cancel.
-    bool scan_silent(const stopper& cancel, const ec_secret& scan_key,
+    /// Invoke callback for each matched tx, false implies cancel.
+    bool scan_silent(const stopper& cancel,
+        const system::silent::batch::receiver& keys,
         const silent_handler& callback) NOEXCEPT;
+    bool scan_silent(const stopper& cancel,
+        const system::silent::batch::receiver& keys, size_t first,
+        size_t last, const silent_handler& callback) NOEXCEPT;
+
+    /// The first silent row at or above first that is not fully written.
+    size_t get_silent_frontier(size_t first) const NOEXCEPT;
 
     /// Verify all signatures in table, false implies cancel.
     bool verify_ecdsa_signatures(const stopper& cancel, header_links&,
@@ -1039,7 +1048,8 @@ class query
         size_t heavy, hash_option&& interval, bool strong) NOEXCEPT;
 
     /// Set silent payment records of a non-coinbase tx.
-    bool set_silent_(const tx_link& link) NOEXCEPT;
+    bool set_silent_(const tx_link& link, const ec_compressed& summary,
+        const system::wallet::silent_payment::scan_outputs& outputs) NOEXCEPT;
 
     /// Pooling.
     /// -----------------------------------------------------------------------

include/bitcoin/database/settings.hpp

@@ -39,7 +39,8 @@ struct BCD_API settings
     settings(system::chain::selection context) NOEXCEPT;
 
     /// Pack the creation envelope, must follow configuration parse.
-    void initialize(const system::settings& bitcoin, bool limited) NOEXCEPT;
+    void initialize(const system::settings& bitcoin, bool limited_blocks,
+        uint32_t silent_start) NOEXCEPT;
 
     /// Table settings.
     /// -----------------------------------------------------------------------

include/bitcoin/database/tables/envelope.hpp

@@ -36,7 +36,7 @@ struct BCD_API envelope
 
     envelope() NOEXCEPT;
     envelope(const system::settings& bitcoin, const settings& database,
-        bool limited) NOEXCEPT;
+        bool limited, uint32_t silent) NOEXCEPT;
 
     void set(const settings& database) NOEXCEPT;
 
@@ -101,6 +101,9 @@ struct BCD_API envelope
     bool limited_blocks{};
     bool provide_filters{};
 
+    /// Node settings.
+    uint32_t silent_start_height{ max_uint32 };
+
     /// Node state.
     bool pooling{};
 };

include/bitcoin/database/tables/schema.hpp

@@ -46,7 +46,8 @@ namespace schema {
 /// 4.0.2.0 Header work reduced to 16 bytes, milestone (formerly merged into
 ///         parent pk) and compact in a new header flags byte.
 /// 4.0.3.0 Second bank of ecdsa, schnorr, and prevalid batch tables.
-constexpr std::array<uint32_t, 4> version{ 4, 0, 3, 0 };
+/// 4.0.4.0 Silent payment start height in the envelope.
+constexpr std::array<uint32_t, 4> version{ 4, 0, 4, 0 };
 
 /// Values.
 /// -----------------------------------------------------------------------

src/settings.cpp

@@ -57,10 +57,10 @@ settings::settings(chain::selection context) NOEXCEPT
     }
 }
 
-void settings::initialize(const system::settings& bitcoin,
-    bool limited) NOEXCEPT
+void settings::initialize(const system::settings& bitcoin, bool limited_blocks,
+    uint32_t silent_start) NOEXCEPT
 {
-    envelope = { bitcoin, *this, limited };
+    envelope = { bitcoin, *this, limited_blocks, silent_start };
 }
 
 } // namespace database

src/tables/envelope.cpp

@@ -55,7 +55,7 @@ void envelope::set(const settings& database) NOEXCEPT
 }
 
 envelope::envelope(const system::settings& bitcoin, const settings& database,
-    bool limited) NOEXCEPT
+    bool limited, uint32_t silent) NOEXCEPT
   : schema(compiled),
     forks(bitcoin.forks),
     initial_subsidy_bitcoin(bitcoin.initial_subsidy_bitcoin),
@@ -87,7 +87,8 @@ envelope::envelope(const system::settings& bitcoin, const settings& database,
     bip9_bit2_active_checkpoint(bitcoin.bip9_bit2_active_checkpoint),
     top_checkpoint(bitcoin.top_checkpoint()),
     milestone(bitcoin.milestone),
-    limited_blocks(limited)
+    limited_blocks(limited),
+    silent_start_height(silent)
 {
     set(database);
 }
@@ -189,6 +190,8 @@ bool envelope::from_data(reader& source) NOEXCEPT
     limited_blocks = to_bool(source.read_byte());
     provide_filters = to_bool(source.read_byte());
 
+    silent_start_height = source.read_little_endian<uint32_t>();
+
     pooling = to_bool(source.read_byte());
     return source;
 }
@@ -279,6 +282,8 @@ bool envelope::to_data(flipper& sink) const NOEXCEPT
     sink.write_byte(to_int<uint8_t>(limited_blocks));
     sink.write_byte(to_int<uint8_t>(provide_filters));
 
+    sink.write_little_endian<uint32_t>(silent_start_height);
+
     sink.write_byte(to_int<uint8_t>(pooling));
     return sink;
 }
@@ -288,7 +293,7 @@ size_t envelope::serialized_size() const NOEXCEPT
     constexpr auto forks_size = 24_size;
     constexpr auto fixed = (4 * sizeof(uint32_t)) + sizeof(uint16_t) +
         (7 * sizeof(uint32_t)) + (10 * sizeof(uint8_t)) + forks_size +
-        sizeof(uint64_t) + (15 * sizeof(uint32_t));
+        sizeof(uint64_t) + (16 * sizeof(uint32_t));
 
     const auto checkpoint_size = [](const chain::checkpoint& in) NOEXCEPT
     {