HIP-1056: Block Streams (#1056)

Signed-off-by: Joseph S <121976561+jsync-swirlds@users.noreply.github.com>
Signed-off-by: Nana Essilfie-Conduah <nana@swirldslabs.com>
Signed-off-by: Edward Wertz <edward@swirldslabs.com>
Signed-off-by: Edward Wertz <123979964+edward-swirldslabs@users.noreply.github.com>
Signed-off-by: Michael Garber <michael.garber@hashgraph.com>
Co-authored-by: Richard Bair <rbair23@users.noreply.github.com>
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Nana Essilfie-Conduah <nana@swirldslabs.com>
Co-authored-by: Edward Wertz <123979964+edward-swirldslabs@users.noreply.github.com>
Co-authored-by: Joseph S. <121976561+jsync-swirlds@users.noreply.github.com>
Co-authored-by: Nana Essilfie-Conduah <56320167+Nana-EC@users.noreply.github.com>
Co-authored-by: Michael Garber <michael.garber@hashgraph.com>

Author: 6 people

HIP/hip-1056.md

@@ -0,0 +1,57 @@
+/**
+ * # Block Stream
+ * The base element of the block stream _at rest_.
+ * A `Block` contains a record of all transactions, results, and outputs for
+ * a block in the chain. Each `Block` also contains a state proof for
+ * validation and a header with version and algorithm information.
+ *
+ * Block entries are not designed for streaming, but for storing blocks in
+ * persistent storage, verifying block stream data, and as query responses
+ * when a block is requested from a block node.
+ *
+ * ### Keywords
+ * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
+ * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
+ * document are to be interpreted as described in
+ * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in
+ * [RFC8174](https://www.ietf.org/rfc/rfc8174).
+ */
+syntax = "proto3";
+
+package com.hedera.hapi.block.stream;
+
+// SPDX-License-Identifier: Apache-2.0
+
+option java_package = "com.hedera.hapi.block.stream.protoc";
+// <<<pbj.java_package = "com.hedera.hapi.block.stream">>> This comment is special code for setting PBJ Compiler java package
+option java_multiple_files = true;
+
+import "stream/block_item.proto";
+
+/**
+ * A single complete Hedera block chain block.
+ *
+ * This is a single block structure and SHALL NOT represent the primary
+ * mechanism to transmit a block stream.<br/>
+ * The primary mechanism for transmitting block stream data SHALL be to
+ * stream individual block items to the block node(s).<br/>
+ * The only delimiter between blocks when streamed SHALL be the `BlockHeader`
+ * item and `BlockProof` item.
+ *
+ * This block SHALL be verifiable as correct using only data in the block,
+ * including the `BlockProof`, and public keys for the consensus nodes.
+ */
+message Block {
+    /**
+     * A list of items that, together, make up this block.
+     * <p>
+     * This list SHALL begin with a `BlockHeader`.<br/>
+     * This list SHALL end with a `BlockProof`.<br/>
+     * Items in this list SHALL be in exactly the same order produced by
+     * consensus.<br/>
+     * Items in this list MAY be filtered, if so requested.<br/>
+     * If this list is filtered, removed items SHALL be replaced with
+     * `FilteredBlockItem` entries.<br/>
+     */
+    repeated BlockItem items = 1;
+}

assets/hip-1056/block-stream-items.svg

@@ -0,0 +1,300 @@
+/**
+ * # Block Item
+ * A single item in the block stream, such as transaction data, event metadata,
+ * or a a system transaction.<br/>
+ * Each block consists of a block header, one or more block items,
+ * and a block state proof. Within the block are a series of events delimited
+ * by start_event block items.
+ *
+ * This structure here MUST support a stream of block items with no enclosing
+ * message.<br/>
+ * Implementations SHOULD behave in a reasonable manner if used in a gRPC
+ * bidirectional streaming RPC similar to
+ * `rpc processBlocks(stream BlockItem) returns (stream Acknowledgement);`.
+ *
+ * ### Keywords
+ * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
+ * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
+ * document are to be interpreted as described in
+ * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in
+ * [RFC8174](https://www.ietf.org/rfc/rfc8174).
+ */
+syntax = "proto3";
+
+package com.hedera.hapi.block.stream;
+
+// SPDX-License-Identifier: Apache-2.0
+
+option java_package = "com.hedera.hapi.block.stream.protoc";
+// <<<pbj.java_package = "com.hedera.hapi.block.stream">>> This comment is special code for setting PBJ Compiler java package
+option java_multiple_files = true;
+
+import "event/event_transaction.proto";
+import "stream/block_proof.proto";
+import "stream/record_file_item.proto";
+import "stream/input/event_metadata.proto";
+import "stream/input/round_header.proto";
+import "stream/output/block_header.proto";
+import "stream/output/state_changes.proto";
+import "stream/output/smart_contract_service.proto";
+import "stream/output/transaction_output.proto";
+import "stream/output/transaction_result.proto";
+
+/**
+ * A single item within a block stream.
+ *
+ * Each item in the block stream SHALL be self-contained and independent,
+ * with the following constraints applicable to the _unfiltered_ stream.
+ * - A block SHALL start with a `header`.
+ * - A block SHALL end with a `state_proof`.
+ * - A `block_header` SHALL be followed by an `event_header`.
+ * - An `event_header` SHALL be followed by one or more
+ *   `event_transaction` items.
+ * - An `event_transaction` SHALL be followed by a `transaction_result`.
+ * - A `transaction_result` MAY be followed by a `transaction_output`.
+ * - A `transaction_result` (or a `transaction_output`, if present) MAY be
+ *     followed by one or more `state_changes`.
+ *
+ * This forms the following required sequence for each block, which is then
+ * repeated within the block stream, indefinitely.  Note that there is no
+ * container structure in the stream, the indentation below is only to
+ * highlight repeated subsequences.<br/>
+ * The order of items within each block below is REQUIRED and SHALL NOT change.
+ *
+ * ```text
+ * header
+ *   repeated {
+ *     start_event
+ *     repeated {
+ *       event_transaction
+ *       transaction_result
+ *       (optional) transaction_output
+ *       (optional) repeated state_changes
+ *     }
+ *   }
+ * state_proof
+ * ```
+ * A filtered stream may exclude some items above, depending on filter
+ * criteria. A filtered item is replaced with a merkle path and hash value
+ * to maintain block stream verifiability.
+ *
+ * A BlockItem SHALL be individually and directly processed to create the
+ * item hash.<br/>
+ * Items to be hashed MUST NOT be contained within another item.<br/>
+ * Items which might be filtered out of the stream MUST NOT be
+ * contained in other items.
+ *
+ * ### Forward Compatibility
+ * In order to maximize forward compatibility, and minimize the need to
+ * coordinate deployments of different systems creating and processing
+ * block streams in the future, the following rules SHALL be followed
+ * for field numbering in this message.
+ * - The first 15 field numbers SHALL be assigned to the fields present
+ *   in the first release. Unused fields in this range SHALL remain reserved
+ *   until needed for additional options that do not fit into "input" or
+ *   "output" categories.
+ * - Fields numbered 16 and above MUST be numbered as follows.
+ *    - "input" items MUST use `odd` field numbers.
+ *    - "output" items MUST use `even` field numbers.
+ *
+ * #### Forward Compatibility Example
+ * A future update adding three new items. A "BlockTrailer" item which is
+ * neither input nor output, a new "ConsensusTransform" which is an input,
+ * and a new "BridgeTransform" which is an output.
+ * - The "BlockTrailer" is field 12, which is removed from the `reserved` list.
+ * - The "ConsensusTransform" is an input, so it is field `17` (the first unused
+ *   `odd` field greater than or equal to 16).
+ * - The "BridgeTransform" is an output, so it is field `16` (the first unused
+ *   even field greater than or equal to 16).
+ *
+ * #### Initial Field assignment to "input", "output", and "other" categories.
+ * - Inputs
+ *    - `event_header`
+ *    - `round_header`
+ *    - `event_transaction`
+ * - Outputs
+ *    - `block_header`
+ *    - `transaction_result`
+ *    - `transaction_output`
+ *    - `state_changes`
+ * - Any subtree (depending on what was filtered).
+ *   This item details it's path in the tree.
+ *    - `filtered_item_hash`
+ * - Neither input nor output (and not part of the "proof" merkle tree)
+ *    - `block_proof`
+ *    - `record_file`
+ */
+message BlockItem {
+    // Reserved for future items that are neither "input" nor "output".
+    reserved 12,13,14,15;
+    oneof item {
+        /**
+         * An header for the block, marking the start of a new block.
+         */
+        com.hedera.hapi.block.stream.output.BlockHeader block_header = 1;
+
+        /**
+         * An header emitted at the start of a new network "event".
+         * <p>
+         * This item SHALL contain the properties relevant to a single
+         * gossip event.
+         */
+        com.hedera.hapi.block.stream.input.EventHeader event_header = 2;
+
+        /**
+         * An header emitted at the start of a new consensus "round".
+         * <p>
+         * This item SHALL contain the properties relevant to a single
+         * consensus round.
+         */
+        com.hedera.hapi.block.stream.input.RoundHeader round_header = 3;
+
+        /**
+         * A single transaction.
+         * <p>
+         * This item SHALL contain the serialized bytes of a
+         * single transaction.<br/>
+         * Each event transaction SHALL be either a `SignedTransaction` or
+         * an internal system-generated transaction.<br/>
+         * This item MUST NOT contain data for more than one
+         * `SignedTransaction` or system-generated transaction.
+         */
+        com.hedera.hapi.platform.event.EventTransaction event_transaction = 4;
+
+        /**
+         * The result of running a transaction.
+         * <p>
+         * This item SHALL be present immediately after an
+         * `event_transaction` item.<br/>
+         * This item MAY be redacted in some circumstances, and SHALL be
+         * replaced with a `filtered_item` if removed.
+         */
+        com.hedera.hapi.block.stream.output.TransactionResult transaction_result = 5;
+
+        /**
+         * A transaction output.
+         * <p>
+         * This item MAY not be present if a transaction does not produce
+         * an output.<br/>
+         * If a transaction does produce an output that is not reflected
+         * in state changes, then this item MUST be present after the
+         * `transaction_result` for that transaction.
+         */
+        com.hedera.hapi.block.stream.output.TransactionOutput transaction_output = 6;
+
+        /**
+         * A set of state changes.
+         * <p>
+         * All changes to values in network state SHALL be described by
+         * stream items of this type.<br/>
+         * The source of these state changes SHALL be described by the
+         * `reason` enumeration.
+         */
+        com.hedera.hapi.block.stream.output.StateChanges state_changes = 7;
+
+        /**
+         * Verification data for items filtered from the stream.<br/>
+         * This is a hash for a merkle tree node where the contents of that
+         * part of the merkle tree have been removed from this stream.
+         * <p>
+         * Items of this type SHALL NOT be present in the full (unfiltered)
+         * block stream.<br/>
+         * Items of this type SHALL replace any item removed from a partial
+         * (filtered) block stream.<br/>
+         * Presence of `filtered_item_hash` entries SHALL NOT prevent
+         * verification of a block, but MAY preclude verification or
+         * reconstruction of consensus state.<br/>
+         */
+        FilteredItemHash filtered_item_hash = 8;
+
+        /**
+         * A signed block proof.<br/>
+         * The signed merkle proof for this block. This will validate
+         * a "virtual" merkle tree containing the previous block "virtual"
+         * root, an "input" subtree, an "output" subtree, and
+         * a "state changes" subtree.
+         * <p>
+         * This item is not part of the block stream hash chain/tree, and
+         * MUST follow after the end of a block.
+         */
+        BlockProof block_proof = 9;
+
+        /**
+         * A record file and associated data.
+         * <p>
+         * This MUST contain a single Record file, associated Sidecar files,
+         * and data from related Signature files.
+         * If this item is present, special treatment is
+         * REQUIRED for this block.
+         * <ul>
+         *   <li>The block SHALL NOT have a `BlockHeader`.</li>
+         *   <li>The block SHALL NOT have a `BlockProof`.</li>
+         *   <li>The block SHALL contain _exactly one_ `RecordFileItem`.</li>
+         *   <li>The block SHALL NOT contain any item other than a
+         *       `RecordFileItem`.</li>
+         *   <li>The content of the `RecordFileItem` MUST be validated using
+         *       the signature data and content provided within according to
+         *       the process used for Record Files prior to the creation
+         *       of Block Stream.</li>
+         * </ul>
+         */
+        RecordFileItem record_file = 10;
+
+        /**
+         * Trace data.
+         */
+        TraceData trace_data = 11;
+    }
+}
+
+/**
+ * Verification data for items filtered from the stream.
+ *
+ * Items of this type SHALL NOT be present in the full (unfiltered) block
+ * stream.<br/>
+ * Items of this type SHALL replace any item removed from a partial (filtered)
+ * block stream.<br/>
+ * Presence of `FilteredItemHash` entries SHALL NOT prevent verification
+ * of a block, but MAY preclude verification or reconstruction
+ * of consensus state.<br/>
+ */
+message FilteredItemHash {
+    /**
+     * A hash of items filtered from the stream.
+     * <p>
+     * The hash algorithm used MUST match the hash algorithm specified in
+     * the block header for the containing block.<br/>
+     * This field is REQUIRED.
+     */
+    bytes hash = 1;
+
+    /**
+     * A binary tree path to a merkle subtree.
+     * This path begins at the root of the block proof merkle tree and ends
+     * at the merkle subtree whose leaf nodes have all been filtered.<br/>
+     * To walk a path `01001` from the root, go left, right, left, left,
+     * then right.
+     * <p>
+     * This REQUIRED field SHALL describe the full path in the virtual
+     * merkle tree constructed for the block proof that contained the
+     * item filtered from the stream.
+     */
+    uint64 filtered_path = 2;
+
+    /**
+     * The log2 value of the number of filtered items.<br/>
+     * Since the filtered merkle subtree is a balanced binary tree, the log2
+     * value of the filtered item count is a whole number.
+     * <p>
+     * The value 2^x where x is this field SHALL be the number of items filtered
+     * in the merkle subtree indicated by the `filtered_path`.
+     */
+    uint64 log2_item_count = 3;
+}
+
+message TraceData {
+    /**
+     * The EVM trace data.
+     */
+    EVMTraceData evm_trace_data = 1;
+}