diff --git a/include/bitcoin/database/impl/query/archive/chain_writer.ipp b/include/bitcoin/database/impl/query/archive/chain_writer.ipp index 95ce1da1b..3957eace2 100644 --- a/include/bitcoin/database/impl/query/archive/chain_writer.ipp +++ b/include/bitcoin/database/impl/query/archive/chain_writer.ipp @@ -214,7 +214,7 @@ code CLASS::set_code(const tx_link& tx_fk, const transaction& tx, if (ad_fk.is_terminal()) return error::tx_address_allocate; - constexpr auto value_parent_diff = sizeof(uint64_t) - tx_link::size; + constexpr auto value_parent = sizeof(uint64_t) - tx_link::size; const auto ptr = store_.address.get_memory(); for (const auto& output: *ous) { @@ -226,7 +226,7 @@ code CLASS::set_code(const tx_link& tx_fk, const transaction& tx, // Calculate next corresponding output fk from serialized size. // (variable_size(value) + (value + script)) - (value - parent) out_fk.value += (variable_size(output->value()) + - output->serialized_size() - value_parent_diff); + output->serialized_size() - value_parent); } } diff --git a/include/bitcoin/database/impl/query/archive/wire_writer.ipp b/include/bitcoin/database/impl/query/archive/wire_writer.ipp index 98c5c1b1e..4e4c93923 100644 --- a/include/bitcoin/database/impl/query/archive/wire_writer.ipp +++ b/include/bitcoin/database/impl/query/archive/wire_writer.ipp @@ -24,8 +24,265 @@ namespace libbitcoin { namespace database { -// TODO: writer directly from wire-encoded buffer (network to store). +// set transaction_view // ---------------------------------------------------------------------------- +// This is the only multitable write query (except initialize/genesis). + +TEMPLATE +code CLASS::set_code(const tx_link& tx_fk, const transaction_view& tx, + bool bypass) NOEXCEPT +{ + using namespace system; + using ix = linkage; + + if (tx.is_empty()) + return error::tx_empty; + + const auto inputs = possible_narrow_cast(tx.inputs()); + const auto outputs = possible_narrow_cast(tx.outputs()); + const auto coinbase = tx.is_coinbase(); + + // ======================================================================== + const auto scope = store_.get_transactor(); + + // Allocate contiguously and store inputs. + input_link in_fk{}; + if (!store_.input.put_link(in_fk, + table::input::put_view{ {}, tx })) + return error::tx_input_put; + + // Allocate contiguously and store outputs. + output_link out_fk{}; + if (!store_.output.put_link(out_fk, + table::output::put_view{ {}, tx_fk, tx })) + return error::tx_output_put; + + // Allocate and contiguously store input links. + ins_link ins_fk{}; + if (!store_.ins.put_link(ins_fk, + table::ins::put_view{ {}, in_fk, tx_fk, tx })) + return error::tx_ins_put; + + // Allocate and contiguously store output links. + outs_link outs_fk{}; + if (!store_.outs.put_link(outs_fk, + table::outs::put_view{ {}, out_fk, tx })) + return error::tx_outs_put; + + // Create tx record. + // Commit is deferred for point/address index consistency. + if (!store_.tx.set(tx_fk, tx.hash(false), table::transaction::put_view + { + {}, + tx, + inputs, + outputs, + ins_fk, + outs_fk + })) + { + return error::tx_tx_set; + } + + // Commit points (hashmap). + if (coinbase) + { + // Should only be one input, but generalized anyway. + if (!store_.point.expand(ins_fk + inputs)) + return error::tx_point_allocate; + + auto source = tx.get_inputs_stream(); + read::bytes::fast ins{ source }; + for (size_t in{}; in < inputs; ++in) + { + // Should always be a null point - but could be invalid. + if (!store_.point.put(ins_fk++, chain::point(ins), + table::point::record{})) + return error::tx_null_point_put; + + // Skip script. + ins.skip_bytes(ins.read_size()); + } + } + else + { + // Expand synchronizes keys with ins_fk, entries set into same offset. + // Allocate contiguous points (at sequential keys matching ins_fk). + if (!store_.point.expand(ins_fk + inputs)) + return error::tx_point_allocate; + + if (store_.is_dirty() || !bypass) + { + // Collect duplicates to store in duplicate table. + std::vector twins{}; + auto ptr = store_.point.get_memory(); + auto source = tx.get_inputs_stream(); + read::bytes::fast ins{ source }; + for (size_t in{}; in < inputs; ++in) + { + bool duplicate{}; + if (!store_.point.put(duplicate, ptr, ins_fk++, + chain::point(ins), table::point::record{})) + return error::tx_point_put; + + if (duplicate) + { + ins.rewind_bytes(chain::point::serialized_size()); + twins.push_back(chain::point(ins)); + } + + // Skip script. + ins.skip_bytes(ins.read_size()); + } + + ptr.reset(); + + // As few duplicates are expected, duplicate domain is only 2^16. + // Return of tx_duplicate_put implies link domain has overflowed. + for (const auto& twin: twins) + if (!store_.duplicate.exists(twin)) + if (!store_.duplicate.put(twin, table::duplicate::record{})) + return error::tx_duplicate_put; + } + else + { + auto ptr = store_.point.get_memory(); + auto source = tx.get_inputs_stream(); + read::bytes::fast ins{ source }; + for (size_t in{}; in < inputs; ++in) + { + if (!store_.point.put(ptr, ins_fk++, chain::point(ins), + table::point::record{})) + return error::tx_point_put; + + // Skip script. + ins.skip_bytes(ins.read_size()); + } + + ptr.reset(); + } + } + + // Commit address index records (hashmap). + if (address_enabled()) + { + auto ad_fk = store_.address.allocate(outputs); + if (ad_fk.is_terminal()) + return error::tx_address_allocate; + + constexpr auto value_parent = sizeof(uint64_t) - tx_link::size; + const auto ptr = store_.address.get_memory(); + auto source = tx.get_outputs_stream(); + read::bytes::fast ous{ source }; + for (size_t out{}; out < outputs; ++out) + { + const auto start = ous.get_read_position(); + const auto value = ous.read_variable(); + if (!store_.address.put(ptr, ad_fk++, + sha256_hash(ous.read_bytes(ous.read_size())), + table::address::record{ {}, out_fk })) + return error::tx_address_put; + + // See outs::put_ref. + // Calculate next corresponding output fk from serialized size. + // (variable_size(value) + (value + script)) - (value - parent) + const auto output_size = ous.get_read_position() - start; + out_fk.value += (variable_size(value) + output_size - value_parent); + } + } + + // Commit tx to search (hashmap). + return store_.tx.commit(tx_fk, tx.hash(false)) ? + error::success : error::tx_tx_commit; + // ======================================================================== +} + +// set txs from block +// ---------------------------------------------------------------------------- +// This sets only the txs of a block with header/context already archived. +// Block MUST be kept in scope until all transactions are written. ~block() +// releases all memory for parts of itself, due to the custom allocator. + +TEMPLATE +code CLASS::set_code(const block_view& block, bool strong, + bool bypass) NOEXCEPT +{ + header_link unused{}; + return set_code(unused, block, strong, bypass); +} + +TEMPLATE +code CLASS::set_code(header_link& out_fk, const block_view& block, bool strong, + bool bypass) NOEXCEPT +{ + out_fk = to_header(block.hash()); + if (out_fk.is_terminal()) + return error::txs_header; + + size_t height{}; + if (!get_height(height, out_fk)) + return error::txs_height; + + return set_code(block, out_fk, strong, bypass, height); +} + +TEMPLATE +code CLASS::set_code(const block_view& block, const header_link& key, + bool strong, bool bypass, size_t height) NOEXCEPT +{ + using namespace system; + if (key.is_terminal()) + return error::txs_header; + + const auto txs = block.transactions(); + if (is_zero(txs)) + return error::txs_empty; + + const auto count = possible_narrow_cast>(txs); + const auto tx_fks = store_.tx.allocate(count); + if (tx_fks.is_terminal()) + return error::tx_tx_allocate; + + code ec{}; + auto fk = tx_fks; + for (const auto& tx: block.views()) + if ((ec = set_code(fk++, tx, bypass))) + return ec; + + // Optional hash, only has value on height intervals. + auto interval = create_interval(key, height); + + // Depth is only set by writer for genesis (is_zero(tx_fks[0])). + const auto depth = store_.interval_depth(); + + using bytes = linkage::integer; + const auto light = possible_narrow_cast( + block.serialized_size(false)); + const auto heavy = possible_narrow_cast + (block.serialized_size(true)); + + // ======================================================================== + const auto scope = store_.get_transactor(); + constexpr auto positive = true; + + // Transactor assures cannot be restored without txs, as required to unset. + if (strong && !set_strong(key, txs, tx_fks, positive)) + return error::txs_confirm; + + // Header link is the key for the txs table. + // Clean single allocation failure (e.g. disk full). + return store_.txs.put(to_txs(key), table::txs::put_group + { + {}, + light, + heavy, + count, + tx_fks, + std::move(interval), + depth + }) ? error::success : error::txs_txs_put; + // ======================================================================== +} } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/query.hpp b/include/bitcoin/database/query.hpp index a340c5d3c..18a708b35 100644 --- a/include/bitcoin/database/query.hpp +++ b/include/bitcoin/database/query.hpp @@ -36,6 +36,7 @@ class query /// Chain type aliases. using block = system::chain::block; + using block_view = system::chain::block_view; using point = system::chain::point; using input = system::chain::input; using output = system::chain::output; @@ -44,6 +45,7 @@ class query using witness = system::chain::witness; using headers = system::chain::header_cptrs; using transaction = system::chain::transaction; + using transaction_view = system::chain::transaction_view; using transactions = system::chain::transaction_cptrs; using inputs_ptr = system::chain::inputs_ptr; using outputs_ptr = system::chain::outputs_ptr; @@ -504,6 +506,13 @@ class query code set_code(const block& block, const header_link& key, bool strong, bool bypass, size_t height) NOEXCEPT; + /// Set block_view (wire). + code set_code(const block_view& block, bool strong, bool bypass) NOEXCEPT; + code set_code(header_link& out_fk, const block_view& block, bool strong, + bool bypass) NOEXCEPT; + code set_code(const block_view& block, const header_link& key, bool strong, + bool bypass, size_t height) NOEXCEPT; + /// Context. /// ----------------------------------------------------------------------- @@ -838,6 +847,8 @@ class query /// ----------------------------------------------------------------------- code set_code(const tx_link& tx_fk, const transaction& tx, bool bypass) NOEXCEPT; + code set_code(const tx_link& tx_fk, const transaction_view& tx, + bool bypass) NOEXCEPT; /// History. /// ----------------------------------------------------------------------- diff --git a/include/bitcoin/database/tables/archives/input.hpp b/include/bitcoin/database/tables/archives/input.hpp index 2c0b50391..8684252d9 100644 --- a/include/bitcoin/database/tables/archives/input.hpp +++ b/include/bitcoin/database/tables/archives/input.hpp @@ -152,7 +152,7 @@ struct input const auto inputs = std::accumulate(ins.cbegin(), ins.cend(), zero, [](size_t total, const auto& in) NOEXCEPT { - // sizes cached, so this is free. + // Includes zero stack size write for non-segregated inputs. return total + in->serialized_size(true); }); @@ -176,6 +176,76 @@ struct input const system::chain::transaction& tx_{}; }; + struct put_view + : public schema::input + { + inline link count() const NOEXCEPT + { + using namespace system; + constexpr auto sequence_size = sizeof(uint32_t); + constexpr auto sequence_point_size = sequence_size + + chain::point::serialized_size(); + + size_t total{}; + + auto istream = tx_.get_inputs_stream(); + read::bytes::fast isource{ istream }; + for (size_t in{}; in < tx_.inputs(); ++in) + { + const auto start = isource.get_read_position(); + isource.skip_bytes(chain::point::serialized_size()); + isource.skip_bytes(isource.read_size() + sequence_size); + const auto input_size = isource.get_read_position() - start; + total += (input_size - sequence_point_size); + } + + if (!tx_.is_segregated()) + { + // Optimize out stream and loop for non-segregated. + total += (tx_.inputs() * variable_size(zero)); + } + else + { + auto wstream = tx_.get_witnesses_stream(); + read::bytes::fast wsource{ wstream }; + for (size_t in{}; in < tx_.inputs(); ++in) + { + const auto stack = wsource.read_size(); + total += variable_size(stack); + for (size_t element{}; element < stack; ++element) + { + const auto element_size = wsource.read_size(); + wsource.skip_bytes(element_size); + total += variable_size(element_size) + element_size; + } + } + } + + return possible_narrow_cast(total); + } + + inline bool to_data(flipper& sink) const NOEXCEPT + { + using namespace system; + auto istream = tx_.get_inputs_stream(); + auto wstream = tx_.get_witnesses_stream(); + read::bytes::fast isource{ istream }; + read::bytes::fast wsource{ wstream }; + for (size_t in{}; in < tx_.inputs(); ++in) + { + // input (without point) + witness stack (or zero). + tx_.write_input_script(sink, isource); + tx_.write_witness(sink, wsource); + } + + BC_ASSERT(isource && wsource); + BC_ASSERT(!sink || sink.get_write_position() == count()); + return sink && isource && wsource; + } + + const system::chain::transaction_view& tx_; + }; + struct wire_script : public schema::input { diff --git a/include/bitcoin/database/tables/archives/ins.hpp b/include/bitcoin/database/tables/archives/ins.hpp index 8d00da0f0..60dfa9d36 100644 --- a/include/bitcoin/database/tables/archives/ins.hpp +++ b/include/bitcoin/database/tables/archives/ins.hpp @@ -133,12 +133,51 @@ struct ins const system::chain::transaction& tx_{}; }; + struct put_view + : public schema::ins + { + inline link count() const NOEXCEPT + { + return system::possible_narrow_cast(tx_.inputs()); + } + + inline bool to_data(flipper& sink) const NOEXCEPT + { + using namespace system; + constexpr auto sequence_point_size = sizeof(uint32_t) + + chain::point::serialized_size(); + + auto in_fk = input_fk; + auto stream = tx_.get_inputs_stream(); + read::bytes::fast source{ stream }; + for (size_t in{}; in < tx_.inputs(); ++in) + { + const auto start = source.get_read_position(); + source.skip_bytes(chain::point::serialized_size()); + source.skip_bytes(source.read_size()); + sink.write_little_endian(source.read_little_endian()); + sink.write_little_endian(in_fk); + sink.write_little_endian(parent_fk); + const auto input_size = source.get_read_position() - start; + in_fk += (input_size - sequence_point_size); + } + + BC_ASSERT(source); + BC_ASSERT(!sink || sink.get_write_position() == count() * minrow); + return sink && source; + } + + const in::integer input_fk{}; + const tx::integer parent_fk{}; + const system::chain::transaction_view& tx_; + }; + struct wire_sequence : public schema::ins { inline bool from_data(reader& source) NOEXCEPT { - const auto sequence_size = sizeof(uint32_t); + constexpr auto sequence_size = sizeof(uint32_t); sink.write_bytes(source.read_bytes(sequence_size)); return source; } diff --git a/include/bitcoin/database/tables/archives/output.hpp b/include/bitcoin/database/tables/archives/output.hpp index 5b1a64fa8..65530eacc 100644 --- a/include/bitcoin/database/tables/archives/output.hpp +++ b/include/bitcoin/database/tables/archives/output.hpp @@ -195,17 +195,15 @@ struct output { using namespace system; static_assert(tx::size <= sizeof(uint64_t)); - constexpr auto value_parent_difference = sizeof(uint64_t) - - tx::size; + constexpr auto value_parent = sizeof(uint64_t) - tx::size; const auto& outs = *tx_.outputs_ptr(); - const auto other = outs.size() * value_parent_difference; + const auto other = outs.size() * value_parent; const auto outputs = std::accumulate(outs.cbegin(), outs.cend(), zero, [](size_t total, const auto& out) NOEXCEPT { - // size cached, so this is free, includes sizeof(value). - return total + variable_size(out->value()) + - out->serialized_size(); + const auto output_size = out->serialized_size(); + return total + variable_size(out->value()) + output_size; }); // Converts value from fixed size wire encoding to variable. @@ -232,6 +230,44 @@ struct output const system::chain::transaction& tx_{}; }; + struct put_view + : public schema::output + { + inline link count() const NOEXCEPT + { + using namespace system; + static_assert(tx::size <= sizeof(uint64_t)); + constexpr auto value_size = sizeof(uint64_t); + constexpr auto value_parent = value_size - tx::size; + + size_t outputs{}; + const auto other = tx_.outputs() * value_parent; + + auto stream = tx_.get_outputs_stream(); + read::bytes::fast source{ stream }; + for (size_t out{}; out < tx_.outputs(); ++out) + { + const auto start = source.get_read_position(); + const auto value = source.read_8_bytes_little_endian(); + source.skip_bytes(value_size + source.read_size()); + const auto output_size = source.get_read_position() - start; + outputs += variable_size(value) + output_size; + } + + // Converts value from fixed size wire encoding to variable. + // (variable_size(value) + (value + script)) - (value - parent) + return possible_narrow_cast(outputs - other); + } + + inline bool to_data(flipper& sink) const NOEXCEPT + { + return sink; + } + + const tx::integer parent_fk{}; + const system::chain::transaction_view& tx_; + }; + struct wire_script : public schema::output { diff --git a/include/bitcoin/database/tables/archives/outs.hpp b/include/bitcoin/database/tables/archives/outs.hpp index 2159f3983..1f8ab024b 100644 --- a/include/bitcoin/database/tables/archives/outs.hpp +++ b/include/bitcoin/database/tables/archives/outs.hpp @@ -107,7 +107,7 @@ struct outs { using namespace system; static_assert(tx::size <= sizeof(uint64_t)); - constexpr auto value_parent_diff = sizeof(uint64_t) - tx::size; + constexpr auto value_parent = sizeof(uint64_t) - tx::size; auto out_fk = output_fk; const auto& outs = *tx_.outputs_ptr(); @@ -119,7 +119,7 @@ struct outs // Calculate next corresponding output fk from serialized size. // (variable_size(value) + (value + script)) - (value - parent) out_fk += (variable_size(out->value()) + - out->serialized_size() - value_parent_diff); + out->serialized_size() - value_parent); }); BC_ASSERT(!sink || sink.get_write_position() == count() * minrow); @@ -129,6 +129,43 @@ struct outs const out::integer output_fk{}; const system::chain::transaction& tx_{}; }; + + struct put_view + : public schema::outs + { + inline link count() const NOEXCEPT + { + return system::possible_narrow_cast(tx_.outputs()); + } + + inline bool to_data(flipper& sink) const NOEXCEPT + { + using namespace system; + static_assert(tx::size <= sizeof(uint64_t)); + constexpr auto value_parent = sizeof(uint64_t) - tx::size; + + auto out_fk = output_fk; + auto stream = tx_.get_outputs_stream(); + read::bytes::fast source{ stream }; + for (size_t out{}; out < tx_.outputs(); ++out) + { + sink.write_little_endian(out_fk); + + const auto start = source.get_read_position(); + const auto value = source.read_variable(); + source.skip_bytes(source.read_size()); + const auto output_size = source.get_read_position() - start; + out_fk += (variable_size(value) + output_size - value_parent); + } + + BC_ASSERT(source); + BC_ASSERT(!sink || sink.get_write_position() == count() * minrow); + return sink && source; + } + + const out::integer output_fk{}; + const system::chain::transaction_view& tx_; + }; }; } // namespace table diff --git a/include/bitcoin/database/tables/archives/transaction.hpp b/include/bitcoin/database/tables/archives/transaction.hpp index b7f720b3e..e1da4e4d5 100644 --- a/include/bitcoin/database/tables/archives/transaction.hpp +++ b/include/bitcoin/database/tables/archives/transaction.hpp @@ -186,6 +186,38 @@ struct transaction outs::integer outs_fk{}; }; + struct put_view + : public schema::transaction + { + inline bool to_data(finalizer& sink) const NOEXCEPT + { + using namespace system; + + // is_coinbase() is computed over the point. + const auto coinbase = tx.is_coinbase(); + const auto light_ = tx.serialized_size(false); + const auto heavy_ = tx.serialized_size(true); + const auto light = possible_narrow_cast(light_); + const auto heavy = possible_narrow_cast(heavy_); + sink.write_little_endian(merge(coinbase, light)); + sink.write_little_endian(heavy); + sink.write_little_endian(tx.locktime()); + sink.write_little_endian(tx.version()); + sink.write_little_endian(ins_count); + sink.write_little_endian(outs_count); + sink.write_little_endian(point_fk); + sink.write_little_endian(outs_fk); + BC_ASSERT(!sink || sink.get_write_position() == minrow); + return sink; + } + + const system::chain::transaction_view& tx; + ix::integer ins_count{}; + ix::integer outs_count{}; + ins::integer point_fk{}; + outs::integer outs_fk{}; + }; + struct only_with_sk : public only { diff --git a/include/bitcoin/database/tables/caches/prevout.hpp b/include/bitcoin/database/tables/caches/prevout.hpp index 86f583b00..1b5abb979 100644 --- a/include/bitcoin/database/tables/caches/prevout.hpp +++ b/include/bitcoin/database/tables/caches/prevout.hpp @@ -85,7 +85,7 @@ struct prevout const auto write_tx = [&](const auto& tx) NOEXCEPT { const auto& ins = tx->inputs_ptr(); - return std::for_each(ins->begin(), ins->end(), + return std::for_each(ins->cbegin(), ins->cend(), [&](const auto& in) NOEXCEPT { const auto value = merge(in->metadata);