Merge pull request #14712 from MinaProtocol/dkijania/revive_extract_d…

…ump_archive_test

Revive extract dump archive test

buildkite/scripts/build-artifact.sh

@@ -23,7 +23,7 @@ make -C src/app/libp2p_helper
 MAINNET_TARGETS=""
 [[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe"
 
-echo "--- Build all major tagets required for packaging"
+echo "--- Build all major targets required for packaging"
 echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
 echo "Rust Version: $(rustc --version)"
 dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
@@ -44,4 +44,5 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
   src/app/rosetta/indexer_test/indexer_test.exe \
   src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
   src/app/test_executive/test_executive.exe  \
-  src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
+  src/test/command_line_tests/command_line_tests.exe \
+  src/test/archive/patch_archive_test/patch_archive_test.exe

buildkite/src/Command/MinaArtifact.dhall

@@ -255,10 +255,10 @@ let docker_step
                     , deb_repo = DebianRepo.Type.Local
                     , deb_profile = spec.profile
                     , step_key =
-                        "test-suite-${DebianVersions.lowerName
-                                        spec.debVersion}${Profiles.toLabelSegment
-                                                            spec.profile}${BuildFlags.toLabelSegment
-                                                                             spec.buildFlags}--docker-image"
+                        "functional_test_suite-${DebianVersions.lowerName
+                                                   spec.debVersion}${Profiles.toLabelSegment
+                                                                       spec.profile}${BuildFlags.toLabelSegment
+                                                                                        spec.buildFlags}-docker-image"
                     , network = "berkeley"
                     }
                   ]

buildkite/src/Command/PatchArchiveTest.dhall

@@ -0,0 +1,30 @@
+let Artifacts = ../Constants/Artifacts.dhall
+
+let Command = ./Base.dhall
+
+let Size = ./Size.dhall
+
+let Network = ../Constants/Network.dhall
+
+let RunWithPostgres = ./RunWithPostgres.dhall
+
+in  { step =
+            \(dependsOn : List Command.TaggedKey.Type)
+        ->  Command.build
+              Command.Config::{
+              , commands =
+                [ RunWithPostgres.runInDockerWithPostgresConn
+                    [ "PATCH_ARCHIVE_TEST_APP=mina-patch-archive-test"
+                    , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db"
+                    ]
+                    "./src/test/archive/sample_db/archive_db.sql"
+                    Artifacts.Type.FunctionalTestSuite
+                    (None Network.Type)
+                    "./scripts/patch-archive-test.sh"
+                ]
+              , label = "Archive: Patch Archive test"
+              , key = "patch-archive-test"
+              , target = Size.Large
+              , depends_on = dependsOn
+              }
+    }

buildkite/src/Jobs/Test/PatchArchiveTest.dhall

@@ -0,0 +1,44 @@
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall
+
+let Profiles = ../../Constants/Profiles.dhall
+
+let Network = ../../Constants/Network.dhall
+
+let Artifacts = ../../Constants/Artifacts.dhall
+
+let Dockers = ../../Constants/DockerVersions.dhall
+
+let dependsOn =
+      Dockers.dependsOn
+        Dockers.Type.Bullseye
+        (None Network.Type)
+        Profiles.Type.Standard
+        Artifacts.Type.FunctionalTestSuite
+
+in  Pipeline.build
+      Pipeline.Config::{
+      , spec = JobSpec::{
+        , dirtyWhen =
+          [ S.strictlyStart (S.contains "src")
+          , S.exactly "scripts/path-archive-test" "sh"
+          , S.exactly "buildkite/src/Jobs/Test/PatchArchiveTest" "dhall"
+          , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall"
+          ]
+        , path = "Test"
+        , name = "PatchArchiveTest"
+        , tags =
+          [ PipelineTag.Type.Long
+          , PipelineTag.Type.Test
+          , PipelineTag.Type.Stable
+          ]
+        }
+      , steps = [ PatchArchiveTest.step dependsOn ]
+      }

scripts/debian/builder-helpers.sh

@@ -241,7 +241,8 @@ build_functional_test_suite_deb() {
 
   # Binaries
   cp ./default/src/test/command_line_tests/command_line_tests.exe "${BUILDDIR}/usr/local/bin/mina-command-line-tests"
-
+  cp ./default/src/test/archive/patch_archive_test/patch_archive_test.exe "${BUILDDIR}/usr/local/bin/mina-patch-archive-test"
+
   build_deb mina-test-suite
 
 }
@@ -364,7 +365,7 @@ build_archive_deb () {
   cp ./default/src/app/extract_blocks/extract_blocks.exe "${BUILDDIR}/usr/local/bin/mina-extract-blocks"
 
   mkdir -p "${BUILDDIR}/etc/mina/archive"
-  cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/etc/mina/archive"
+  cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/usr/local/bin/mina-missing-blocks-guardian"
 
   cp ./default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe "${BUILDDIR}/usr/local/bin/mina-missing-blocks-auditor"
   cp ./default/src/app/replayer/replayer.exe "${BUILDDIR}/usr/local/bin/mina-replayer"

scripts/patch-archive-test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+# test replayer on known archive db
+
+NETWORK_DATA_FOLDER=src/test/archive/sample_db
+PATCH_ARCHIVE_TEST_APP=${PATCH_ARCHIVE_TEST_APP:-_build/default/src/test/archive/patch_archive_test/patch_archive_test.exe}
+PG_PORT=${PG_PORT:-5432}
+POSTGRES_USER=${POSTGRES_USER:-postgres}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+
+CONN=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${PG_PORT}
+
+
+echo "Running patch archive test"
+$PATCH_ARCHIVE_TEST_APP --source-uri $CONN \
+                        --network-data-folder $NETWORK_DATA_FOLDER

scripts/replayer-test.sh

@@ -5,7 +5,7 @@ set -x
 
 INPUT_FILE=src/test/archive/sample_db/replayer_input_file.json
 REPLAYER_APP=_build/default/src/app/replayer/replayer.exe
-PG_CONN=postgres://postgres:postgres@localhost:5433/archive
+PG_CONN=postgres://postgres:postgres@localhost:5432/archive
 
 while [[ "$#" -gt 0 ]]; do case $1 in
   -i|--input-file) INPUT_FILE="$2"; shift;;

src/dune-project

@@ -136,6 +136,7 @@
 (package (name parallel_scan))
 (package (name participating_state))
 (package (name pasta_bindings))
+(package (name patch_archive_test))
 (package (name perf_histograms))
 (package (name pickles_base))
 (package (name pickles))

src/test/archive/patch_archive_test/dune

@@ -0,0 +1,31 @@
+(executable
+ (package patch_archive_test)
+ (name patch_archive_test)
+ (public_name patch_archive_test)
+ (libraries
+   async
+   async.async_command
+   core_kernel
+   caqti
+   caqti-async
+   caqti-driver-postgresql
+   integration_test_lib
+   archive_lib
+   block_time
+   mina_numbers
+   logger
+   mina_base
+   uri
+   base
+   async_kernel
+   core
+   async_unix
+   stdio
+   base.caml
+   result
+   mina_automation
+   bounded_types
+ )
+ (preprocessor_deps ../../../config.mlh)
+ (instrumentation (backend bisect_ppx))
+ (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv h_list.ppx)))

src/test/archive/patch_archive_test/patch_archive_test.ml

@@ -0,0 +1,163 @@
+(* patch_archive_test.ml *)
+
+(* test patching of archive databases
+
+   test structure:
+    - import reference database for comparision (for example with 100 blocks)
+    - create new schema and export blocks from reference db with some missing ones
+    - patch the database with missing precomputed blocks
+    - compare original and copy
+*)
+
+module Network_Data = struct
+  type t =
+    { init_script : String.t
+    ; precomputed_blocks_zip : String.t
+    ; genesis_ledger_file : String.t
+    ; replayer_input_file : String.t
+    ; folder : String.t
+    }
+
+  let create folder =
+    { init_script = "archive_db.sql"
+    ; genesis_ledger_file = "input.json"
+    ; precomputed_blocks_zip = "precomputed_blocks.zip"
+    ; replayer_input_file = "replayer_input_file.json"
+    ; folder
+    }
+end
+
+open Core_kernel
+open Async
+open Mina_automation
+
+let main ~db_uri ~network_data_folder () =
+  let open Deferred.Let_syntax in
+  let missing_blocks_count = 3 in
+  let network_name = "dummy" in
+
+  let network_data = Network_Data.create network_data_folder in
+
+  let output_folder = Filename.temp_dir_name ^ "/output" in
+
+  let%bind output_folder = Unix.mkdtemp output_folder in
+
+  let connection = Psql.Conn_str db_uri in
+
+  let source_db_name = "patch_archive_test_source" in
+  let target_db_name = "patch_archive_test_target" in
+  let%bind _ = Psql.create_empty_db ~connection ~db:source_db_name in
+  let%bind _ =
+    Psql.run_script ~connection ~db:source_db_name
+      (network_data.folder ^ "/" ^ network_data.init_script)
+  in
+  let%bind () = Psql.create_mina_db ~connection ~db:target_db_name in
+
+  let source_db = db_uri ^ "/" ^ source_db_name in
+  let target_db = db_uri ^ "/" ^ target_db_name in
+
+  let extract_blocks = Extract_blocks.of_context Executor.AutoDetect in
+  let config =
+    { Extract_blocks.Config.archive_uri = source_db
+    ; range = Extract_blocks.Config.AllBlocks
+    ; output_folder = Some output_folder
+    ; network = Some network_name
+    ; include_block_height_in_name = true
+    }
+  in
+  let%bind _ = Extract_blocks.run extract_blocks ~config in
+
+  let archive_blocks = Archive_blocks.of_context Executor.AutoDetect in
+
+  let%bind extensional_files =
+    Sys.ls_dir output_folder
+    >>= Deferred.List.map ~f:(fun e ->
+            Deferred.return (output_folder ^ "/" ^ e) )
+  in
+
+  let n =
+    List.init missing_blocks_count ~f:(fun _ ->
+        (* never remove last block as missing-block-guardian can have issues when patching it
+           as it patching only gaps
+        *)
+        Random.int (List.length extensional_files - 1) )
+  in
+
+  let unpatched_extensional_files =
+    List.filteri extensional_files ~f:(fun i _ ->
+        not (List.mem n i ~equal:Int.equal) )
+    |> List.dedup_and_sort ~compare:(fun left right ->
+           let scan_height item =
+             let item =
+               Filename.basename item |> Str.global_replace (Str.regexp "-") " "
+             in
+             Scanf.sscanf item "%s %d %s" (fun _ height _ -> height)
+           in
+
+           let left_height = scan_height left in
+           let right_height = scan_height right in
+
+           Int.compare left_height right_height )
+  in
+
+  let%bind _ =
+    Archive_blocks.run archive_blocks ~blocks:unpatched_extensional_files
+      ~archive_uri:target_db ~format:Extensional
+  in
+
+  let%bind missing_blocks_auditor_path =
+    Missing_blocks_auditor.of_context Executor.AutoDetect
+    |> Missing_blocks_auditor.path
+  in
+
+  let%bind archive_blocks_path = Archive_blocks.path archive_blocks in
+
+  let config =
+    { Missing_blocks_guardian.Config.archive_uri = Uri.of_string target_db
+    ; precomputed_blocks = Uri.make ~scheme:"file" ~path:output_folder ()
+    ; network = network_name
+    ; run_mode = Run
+    ; missing_blocks_auditor = missing_blocks_auditor_path
+    ; archive_blocks = archive_blocks_path
+    ; block_format = Extensional
+    }
+  in
+
+  let missing_blocks_guardian =
+    Missing_blocks_guardian.of_context Executor.AutoDetect
+  in
+
+  let%bind _ = Missing_blocks_guardian.run missing_blocks_guardian ~config in
+
+  let replayer = Replayer.of_context Executor.AutoDetect in
+
+  let%bind _ =
+    Replayer.run replayer ~archive_uri:target_db
+      ~input_config:
+        (network_data.folder ^ "/" ^ network_data.replayer_input_file)
+      ~interval_checkpoint:10 ~output_ledger:"./output_ledger" ()
+  in
+
+  Deferred.unit
+
+let () =
+  Command.(
+    run
+      (let open Let_syntax in
+      async ~summary:"Test patching of blocks in an archive database"
+        (let%map db_uri =
+           Param.flag "--source-uri"
+             ~doc:
+               "URI URI for connecting to the database (e.g., \
+                postgres://$USER@localhost:5432)"
+             Param.(required string)
+         and network_data_folder =
+           Param.(
+             flag "--network-data-folder" ~aliases:[ "network-data-folder" ]
+               Param.(required string))
+             ~doc:
+               "Path Path to folder containing network data. Usually it's sql \
+                for db import, genesis ledger and zipped precomputed blocks \
+                archive"
+         in
+         main ~db_uri ~network_data_folder )))

src/test/mina_automation/archive_blocks.ml

@@ -9,7 +9,7 @@ include Executor
 let of_context context =
   Executor.of_context ~context
     ~dune_name:"src/app/archive_blocks/archive_blocks.exe"
-    ~official_name:"/usr/local/bin/mina-archive-blocks"
+    ~official_name:"mina-archive-blocks"
 
 type format = Precomputed | Extensional