Merge blockbuster into DAS (#189)

* Add blockbuster

* Fix gitignore

* Auto fix clippy

* Fix clippy

* Fix cargo fmt

.gitignore

@@ -19,7 +19,6 @@ dist
 test-ledger
 *.swp
 *error.log
-programs
 *.iml
 skaffold-state.json
 test-programs

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+  "blockbuster",
   "core",
   "das_api",
   "digital_asset_types",
@@ -31,17 +32,18 @@ anyhow = "1.0.75"
 async-std = "1.0.0"
 async-trait = "0.1.60"
 backon = "0.4.1"
-blockbuster = "2.3.0"
+blockbuster = {path = "blockbuster"}
 borsh = "~0.10.3"
 borsh-derive = "~0.10.3"
 bs58 = "0.4.0"
+bytemuck = {version = "1.14.0", features = ["derive"]}
 cadence = "0.29.0"
 cadence-macros = "0.29.0"
 chrono = "0.4.19"
 clap = "4.2.2"
-das_api = { path = "das_api" }
-das-core = { path = "core" }
-digital_asset_types = { path = "digital_asset_types" }
+das-core = {path = "core"}
+das_api = {path = "das_api"}
+digital_asset_types = {path = "digital_asset_types"}
 enum-iterator = "1.2.0"
 enum-iterator-derive = "1.1.0"
 env_logger = "0.10.0"
@@ -63,19 +65,20 @@ jsonrpsee-core = "0.16.2"
 lazy_static = "1.4.0"
 log = "0.4.17"
 metrics = "0.20.1"
-migration = { path = "migration" }
+migration = {path = "migration"}
 mime_guess = "2.0.4"
 mpl-bubblegum = "1.2.0"
+mpl-core = {version = "0.5.0", features = ["serde"]}
 mpl-token-metadata = "4.1.1"
-nft_ingester = { path = "nft_ingester" }
+nft_ingester = {path = "nft_ingester"}
 num-derive = "0.3.3"
 num-traits = "0.2.15"
 once_cell = "1.19.0"
 open-rpc-derive = "0.0.4"
 open-rpc-schema = "0.0.4"
 plerkle_messenger = "1.6.0"
 plerkle_serialization = "1.8.0"
-program_transformers = { path = "program_transformers" }
+program_transformers = {path = "program_transformers"}
 prometheus = "0.13.3"
 proxy-wasm = "0.2.0"
 rand = "0.8.5"
@@ -92,14 +95,20 @@ serde_json = "1.0.81"
 serial_test = "2.0.0"
 solana-account-decoder = "~1.17"
 solana-client = "~1.17"
+solana-geyser-plugin-interface = "~1.17"
 solana-program = "~1.17"
 solana-sdk = "~1.17"
 solana-transaction-status = "~1.17"
+solana-zk-token-sdk = "1.17.16"
 spl-account-compression = "0.3.0"
 spl-associated-token-account = ">= 1.1.3, < 3.0"
 spl-concurrent-merkle-tree = "0.2.0"
 spl-noop = "0.2.0"
+spl-pod = {version = "0.1.0", features = ["serde-traits"]}
 spl-token = ">= 3.5.0, < 5.0"
+spl-token-2022 = {version = "1.0", features = ["no-entrypoint"]}
+spl-token-group-interface = "0.1.0"
+spl-token-metadata-interface = "0.2.0"
 sqlx = "0.6.2"
 stretto = "0.7.2"
 thiserror = "1.0.31"
@@ -109,7 +118,7 @@ tower = "0.4.13"
 tower-http = "0.3.5"
 tracing = "0.1.35"
 tracing-subscriber = "0.3.16"
-txn_forwarder = { path = "tools/txn_forwarder" }
+txn_forwarder = {path = "tools/txn_forwarder"}
 url = "2.3.1"
 wasi = "0.7.0"
 wasm-bindgen = "0.2.83"

blockbuster/.gitignore

@@ -0,0 +1,11 @@
+# Generated by Cargo
+# will have compiled files and executables
+/target/
+
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+*.iml
+blockbuster/target
+target/

blockbuster/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+authors = ["Metaplex Developers <[email protected]>"]
+description = "Metaplex canonical program parsers, for indexing, analytics etc...."
+edition = "2021"
+license = "AGPL-3.0"
+name = "blockbuster"
+readme = "../README.md"
+repository = "https://github.com/metaplex-foundation/blockbuster"
+version = "2.3.0"
+
+[dependencies]
+anchor-lang = {workspace = true}
+async-trait = {workspace = true}
+borsh = {workspace = true}
+bs58 = {workspace = true}
+bytemuck = {workspace = true}
+lazy_static = {workspace = true}
+log = {workspace = true}
+mpl-bubblegum = {workspace = true}
+mpl-core = {workspace = true, features = ["serde"]}
+mpl-token-metadata = {workspace = true, features = ["serde"]}
+serde = {workspace = true}
+solana-sdk = {workspace = true}
+solana-transaction-status = {workspace = true}
+solana-zk-token-sdk = {workspace = true}
+spl-account-compression = {workspace = true, features = ["no-entrypoint"]}
+spl-noop = {workspace = true, features = ["no-entrypoint"]}
+spl-pod = {workspace = true, features = ["serde-traits"]}
+spl-token = {workspace = true, features = ["no-entrypoint"]}
+spl-token-2022 = {workspace = true, features = ["no-entrypoint"]}
+spl-token-group-interface = {workspace = true}
+spl-token-metadata-interface = {workspace = true}
+thiserror = {workspace = true}
+
+[dev-dependencies]
+flatbuffers = {workspace = true}
+plerkle_serialization = {workspace = true}
+rand = {workspace = true}
+serde_json = {workspace = true}
+solana-client = {workspace = true}
+solana-geyser-plugin-interface = {workspace = true}
+spl-concurrent-merkle-tree = {workspace = true}

blockbuster/README.md

@@ -0,0 +1,22 @@
+# BlockBuster
+
+BlockBuster -> "Busting Solana blocks into little pieces to index and operate on the programs therein" - Noone - 1995
+
+This repository is the home for Metaplex Program Parsers. Program parsers are canonical libraries that take a transaction or account update from a geyser plugin and parse them correctly according to Metaplex smart contracts. This sort of parsing is hard to automate as it must contain some knowledge of the API structure of the contract which is not fully describable yet via IDLs. Things like remaining accounts, optional accounts and complex instruction data are not always 100% clear what they mean without knowledge of the contract. 
+
+## Mode of Operation
+This library works best as a consumer of messages sent via a geyser plugin using the [Plerkle Serialization](https://github.com/metaplex-foundation/digital-asset-validator-plugin) library by metaplex. The types from that library are FlatBuffer based currently, and are the wire format of messages coming out of Plerkle into the rest of the infrastructure.
+For more information about Plerkle and the [Digital Asset RPC infrastructure](https://github.com/metaplex-foundation/digital-asset-validator-plugin) It can however be used in any general programs provided you can create the data in the FlatBuffer types.
+
+## Scope
+
+This library contains parsers for the following programs and the parsers are specific to how these contracts relate to metaplex assets.
+
+* Gummyroll (Solana)
+* Bubblegum (Metaplex)
+* Spl Token (Solana)
+* Token Metadata (Metaplex)
+* Auction House (Metaplex)
+* Candy Machine (Metaplex)
+* Hydra (Metaplex)
+

blockbuster/src/error.rs

@@ -0,0 +1,34 @@
+use std::io::Error;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum BlockbusterError {
+    #[error("Instruction Data Parsing Error")]
+    InstructionParsingError,
+    #[error("IO Error {0}")]
+    IOError(String),
+    #[error("Could not deserialize data")]
+    DeserializationError,
+    #[error("Missing Bubblegum event data")]
+    MissingBubblegumEventData,
+    #[error("Data length is invalid.")]
+    InvalidDataLength,
+    #[error("Unknown anchor account discriminator.")]
+    UnknownAccountDiscriminator,
+    #[error("Account type is not valid")]
+    InvalidAccountType,
+    #[error("Master edition version is invalid")]
+    FailedToDeserializeToMasterEdition,
+    #[error("Uninitialized account type")]
+    UninitializedAccount,
+    #[error("Account Type Not implemented")]
+    AccountTypeNotImplemented,
+    #[error("Could not deserialize data: {0}")]
+    CustomDeserializationError(String),
+}
+
+impl From<std::io::Error> for BlockbusterError {
+    fn from(err: Error) -> Self {
+        BlockbusterError::IOError(err.to_string())
+    }
+}

blockbuster/src/instruction.rs

@@ -0,0 +1,89 @@
+use solana_sdk::{instruction::CompiledInstruction, pubkey::Pubkey};
+use solana_transaction_status::InnerInstructions;
+use std::collections::{HashSet, VecDeque};
+
+pub type IxPair<'a> = (Pubkey, &'a CompiledInstruction);
+
+#[derive(Debug, Clone, Copy)]
+pub struct InstructionBundle<'a> {
+    pub txn_id: &'a str,
+    pub program: Pubkey,
+    pub instruction: Option<&'a CompiledInstruction>,
+    pub inner_ix: Option<&'a [IxPair<'a>]>,
+    pub keys: &'a [Pubkey],
+    pub slot: u64,
+}
+
+impl<'a> Default for InstructionBundle<'a> {
+    fn default() -> Self {
+        InstructionBundle {
+            txn_id: "",
+            program: Pubkey::new_from_array([0; 32]),
+            instruction: None,
+            inner_ix: None,
+            keys: &[],
+            slot: 0,
+        }
+    }
+}
+
+pub fn order_instructions<'a>(
+    programs: &HashSet<Pubkey>,
+    account_keys: &[Pubkey],
+    message_instructions: &'a [CompiledInstruction],
+    meta_inner_instructions: &'a [InnerInstructions],
+) -> VecDeque<(IxPair<'a>, Option<Vec<IxPair<'a>>>)> {
+    let mut ordered_ixs: VecDeque<(IxPair, Option<Vec<IxPair>>)> = VecDeque::new();
+
+    // Get inner instructions.
+    for (outer_instruction_index, message_instruction) in message_instructions.iter().enumerate() {
+        let non_hoisted_inner_instruction = meta_inner_instructions
+            .iter()
+            .filter_map(|ix| {
+                (ix.index == outer_instruction_index as u8).then_some(&ix.instructions)
+            })
+            .flatten()
+            .map(|inner_ix| {
+                let cix = &inner_ix.instruction;
+                (account_keys[cix.program_id_index as usize], cix)
+            })
+            .collect::<Vec<IxPair>>();
+
+        let hoisted = hoist_known_programs(programs, &non_hoisted_inner_instruction);
+        ordered_ixs.extend(hoisted);
+
+        if let Some(outer_program_id) =
+            account_keys.get(message_instruction.program_id_index as usize)
+        {
+            if programs.contains(outer_program_id) {
+                ordered_ixs.push_back((
+                    (*outer_program_id, message_instruction),
+                    Some(non_hoisted_inner_instruction),
+                ));
+            }
+        } else {
+            eprintln!("outer program id deserialization error");
+        }
+    }
+    ordered_ixs
+}
+
+fn hoist_known_programs<'a>(
+    programs: &HashSet<Pubkey>,
+    ix_pairs: &[IxPair<'a>],
+) -> Vec<(IxPair<'a>, Option<Vec<IxPair<'a>>>)> {
+    ix_pairs
+        .iter()
+        .enumerate()
+        .filter(|&(_index, &(pid, _ci))| programs.contains(&pid))
+        .map(|(index, &(pid, ci))| {
+            let inner_copy = ix_pairs
+                .iter()
+                .skip(index + 1)
+                .take_while(|&&(inner_pid, _)| inner_pid != pid)
+                .cloned()
+                .collect::<Vec<IxPair<'a>>>();
+            ((pid, ci), Some(inner_copy))
+        })
+        .collect()
+}

blockbuster/src/lib.rs

@@ -0,0 +1,7 @@
+pub mod error;
+pub mod instruction;
+pub mod program_handler;
+pub mod programs;
+
+pub use mpl_core;
+pub use mpl_token_metadata as token_metadata;

blockbuster/src/parsed_programs.rs

@@ -0,0 +1,9 @@
+pub enum Program {
+    Bubblegum {
+        parser: bubblegum::BubblegumParser,
+        instruction_result: BubblegumInstruction,
+        account_result: (),
+    },
+}
+
+impl ProgramParser for Program {}