Port multi generic agents to develop

Makefile

@@ -132,11 +132,6 @@ replayer: ocaml_checks
 	ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/replayer/replayer.exe --profile=devnet
 	$(info Build complete)
 
-replayer: ocaml_checks
-	$(info Starting Build)
-	ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/replayer/replayer.exe --profile=devnet
-	$(info Build complete)
-
 missing_blocks_auditor: ocaml_checks
 	$(info Starting Build)
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/missing_blocks_auditor/missing_blocks_auditor.exe --profile=testnet_postake_medium_curves

buildkite/scripts/check-compatibility.sh

@@ -6,7 +6,7 @@
 set +e
 
 function get_shas {
-  SHAS=$(git log -n 10 --format="%h" --abbrev=7 --no-merges)
+  SHAS=$(git log -n 10 --format="%h" --abbrev=7 --first-parent)
 }
 
 function image_tag {

buildkite/scripts/generate-genesis-config.sh

@@ -24,7 +24,7 @@ mkdir hardfork_ledgers
 mina-create-genesis --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json | tee runtime_genesis_ledger.log | mina-logproc
 
 echo "--- Create hardfork config"
-FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json mina-hf-create-runtime-config > new_config.json
+FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json
 
 echo "--- New genesis config"
 jq 'del(.ledger.s3_data_hash, .epoch_data.staking.s3_data_hash, .epoch_data.next.s3_data_hash)' new_config.json

buildkite/src/Command/Base.dhall

@@ -118,6 +118,7 @@ let Config =
           , skip = None B/Skip
           , if = None B/If
           , timeout_in_minutes = None Integer
+          , target = Size.Multi
           }
       }
 
@@ -131,6 +132,7 @@ let targetToAgent =
             , Integration = toMap { size = "integration" }
             , QA = toMap { size = "qa" }
             , Hardfork = toMap { size = "hardfork" }
+            , Multi = toMap { size = "generic-multi" }
             }
             target
 

buildkite/src/Command/CheckGraphQLSchema.dhall

@@ -1,7 +1,5 @@
 let Command = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let RunInToolchain = ./RunInToolchain.dhall
 
 in  { step =
@@ -14,7 +12,6 @@ in  { step =
                     "./buildkite/scripts/check-graphql-schema.sh"
               , label = "Check GraphQL Schema"
               , key = "check-graphql-schema"
-              , target = Size.Large
               , depends_on = dependsOn
               }
     }

buildkite/src/Command/ConnectToTestnet.dhall

@@ -2,8 +2,6 @@ let B = ../External/Buildkite.dhall
 
 let Command = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let RunInToolchain = ./RunInToolchain.dhall
 
 let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
@@ -23,7 +21,6 @@ in  { step =
               , label = "Connect to ${testnet}"
               , soft_fail = Some soft_fail
               , key = "connect-to-${testnet}"
-              , target = Size.Large
               , depends_on = dependsOn
               }
     }

buildkite/src/Command/DeployTestnet.dhall

@@ -1,7 +1,5 @@
 let Command = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let Cmd = ../Lib/Cmds.dhall
 
 let DeploySpec =
@@ -49,7 +47,6 @@ in  { step =
                 ]
               , label = "Deploy ${spec.testnetLabel}"
               , key = "deploy-${spec.testnetLabel}"
-              , target = Size.Large
               , depends_on = spec.deps
               }
     , DeploySpec = DeploySpec

buildkite/src/Command/HardforkPackageGeneration.dhall

@@ -22,8 +22,6 @@ let PipelineMode = ../Pipeline/Mode.dhall
 
 let JobSpec = ../Pipeline/JobSpec.dhall
 
-let Size = ./Size.dhall
-
 let DockerImage = ./DockerImage.dhall
 
 let DebianVersions = ../Constants/DebianVersions.dhall
@@ -145,7 +143,6 @@ let pipeline
                         "Build Mina Hardfork Package for ${DebianVersions.capitalName
                                                              debVersion}"
                     , key = generateLedgersJobKey
-                    , target = Size.XLarge
                     }
                 , Command.build
                     Command.Config::{
@@ -164,7 +161,6 @@ let pipeline
                     , depends_on =
                       [ { name = pipelineName, key = generateLedgersJobKey } ]
                     , key = "publish-hardfork-deb-pkg"
-                    , target = Size.Small
                     }
                 , DockerImage.generateStep dockerSpec
                 , Command.build
@@ -177,7 +173,6 @@ let pipeline
                     , label =
                         "Assert corrupted packaged artifacts are unverifiable"
                     , key = "assert-unverify-corrupted-packaged-artifacts"
-                    , target = Size.XLarge
                     , depends_on =
                       [ { name = pipelineName
                         , key = DockerImage.stepKey dockerSpec
@@ -194,7 +189,6 @@ let pipeline
                       ]
                     , label = "Verify packaged artifacts"
                     , key = "verify-packaged-artifacts"
-                    , target = Size.XLarge
                     , depends_on =
                       [ { name = pipelineName
                         , key = DockerImage.stepKey dockerSpec

buildkite/src/Command/Libp2pHelperBuild.dhall

@@ -1,7 +1,5 @@
 let Command = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let Toolchain = ../Constants/Toolchain.dhall
 
 let BuildFlags = ../Constants/BuildFlags.dhall
@@ -35,7 +33,6 @@ let cmdConfig =
                                              debVersion} ${BuildFlags.toSuffixUppercase
                                                              buildFlags}"
             , key = "libp2p-helper${BuildFlags.toLabelSegment buildFlags}"
-            , target = Size.Small
             }
 
 in  { step = cmdConfig }

buildkite/src/Command/Mina.dhall

@@ -4,8 +4,6 @@ let Docker = ./Docker/Type.dhall
 
 let Base = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let dockerImage = (../Constants/ContainerImages.dhall).minaToolchain
 
 let fixPermissionsScript = "sudo chown -R opam ."
@@ -23,7 +21,6 @@ let build
             , commands = [ Cmd.run fixPermissionsScript ] # c.commands
             , label = c.label
             , key = c.key
-            , target = Size.Small
             , docker = Some Docker::{ image = dockerImage }
             }
 

buildkite/src/Command/MinaArtifact.dhall

@@ -14,8 +14,6 @@ let PipelineMode = ../Pipeline/Mode.dhall
 
 let JobSpec = ../Pipeline/JobSpec.dhall
 
-let Size = ./Size.dhall
-
 let Libp2p = ./Libp2pHelperBuild.dhall
 
 let DockerImage = ./DockerImage.dhall
@@ -116,7 +114,6 @@ let build_artifacts
                   ]
             , label = "Debian: Build ${labelSuffix spec}"
             , key = "build-deb-pkg"
-            , target = Size.XLarge
             , retries =
               [ Command.Retry::{
                 , exit_status = Command.ExitStatus.Code +2
@@ -285,7 +282,6 @@ let publish_to_debian_repo =
                   spec.profile
                   spec.buildFlags
                   "build"
-            , target = Size.Small
             }
 
 let pipelineBuilder

buildkite/src/Command/Promotion/PromoteDebian.dhall

@@ -16,8 +16,6 @@ let Toolchain = ../../Constants/Toolchain.dhall
 
 let Command = ../Base.dhall
 
-let Size = ../Size.dhall
-
 let PromoteDebianSpec =
       { Type =
           { deps : List Command.TaggedKey.Type
@@ -82,7 +80,6 @@ let promoteDebianStep =
                                                                                                                                                                                                                                                                                                                                   spec.to_channel} ${new_name}"
                 , label = "Debian: ${spec.step_key}"
                 , key = spec.step_key
-                , target = Size.Small
                 , depends_on = spec.deps
                 , if = spec.if
                 }

buildkite/src/Command/Promotion/PromoteDocker.dhall

@@ -16,8 +16,6 @@ let DebianVersions = ../../Constants/DebianVersions.dhall
 
 let Command = ../Base.dhall
 
-let Size = ../Size.dhall
-
 let Cmd = ../../Lib/Cmds.dhall
 
 let PromoteDockerSpec =
@@ -87,7 +85,6 @@ let promoteDockerStep =
                 , commands = commands
                 , label = "Docker: ${spec.step_key}"
                 , key = spec.step_key
-                , target = Size.XLarge
                 , depends_on = spec.deps
                 , if = spec.if
                 }

buildkite/src/Command/Promotion/VerifyDebian.dhall

@@ -8,8 +8,6 @@ let DebianVersions = ../../Constants/DebianVersions.dhall
 
 let Command = ../Base.dhall
 
-let Size = ../Size.dhall
-
 let Cmd = ../../Lib/Cmds.dhall
 
 let PromoteDebian = ./PromoteDebian.dhall
@@ -36,7 +34,6 @@ let promoteDebianVerificationStep =
                   ]
                 , label = "Debian: ${spec.step_key}"
                 , key = spec.step_key
-                , target = Size.Small
                 , depends_on = spec.deps
                 , if = spec.if
                 }

buildkite/src/Command/Promotion/VerifyDocker.dhall

@@ -6,8 +6,6 @@ let List/map = Prelude.List.map
 
 let Command = ../Base.dhall
 
-let Size = ../Size.dhall
-
 let Cmd = ../../Lib/Cmds.dhall
 
 let PromoteDocker = ./PromoteDocker.dhall
@@ -46,7 +44,6 @@ let promoteDockerVerificationStep =
                 , commands = commands
                 , label = "Docker: ${spec.step_key}"
                 , key = spec.step_key
-                , target = Size.Small
                 , depends_on = spec.deps
                 , if = spec.if
                 }

buildkite/src/Command/ReplayerTest.dhall

@@ -2,8 +2,6 @@ let Artifacts = ../Constants/Artifacts.dhall
 
 let Command = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let RunWithPostgres = ./RunWithPostgres.dhall
 
 let Network = ../Constants/Network.dhall
@@ -22,7 +20,6 @@ in  { step =
                 ]
               , label = "Archive: Replayer test"
               , key = "replayer-test"
-              , target = Size.Large
               , depends_on = dependsOn
               }
     }

buildkite/src/Command/Size.dhall

@@ -1 +1 @@
-< XLarge | Large | Medium | Small | Integration | QA | Hardfork >
+< XLarge | Large | Medium | Small | Multi | Integration | QA | Hardfork >

buildkite/src/Command/TestExecutive.dhall

@@ -1,9 +1,9 @@
 let Command = ./Base.dhall
 
-let Size = ./Size.dhall
-
 let Cmd = ../Lib/Cmds.dhall
 
+let Size = ../Command/Size.dhall
+
 let SelectFiles = ../Lib/SelectFiles.dhall
 
 in  { executeCloud =
@@ -18,8 +18,8 @@ in  { executeCloud =
               , artifact_paths =
                 [ SelectFiles.exactly "." "${testName}.cloud.test.log" ]
               , label = "${testName} integration test"
+              , target = Size.XLarge
               , key = "integration-test-${testName}"
-              , target = Size.Integration
               , depends_on = dependsOn
               }
     , executeLocal =

buildkite/src/Entrypoints/GenerateHardforkPackage.dhall

@@ -13,8 +13,6 @@ let JobSpec = ../Pipeline/JobSpec.dhall
 
 let Pipeline = ../Pipeline/Dsl.dhall
 
-let Size = ../Command/Size.dhall
-
 let HardforkPackageGeneration = ../Command/HardforkPackageGeneration.dhall
 
 let generate_hardfork_package =
@@ -39,7 +37,6 @@ let generate_hardfork_package =
                         ]
                       , label = "Generate hardfork package"
                       , key = "generate-hardfork-package"
-                      , target = Size.Small
                       , docker = Some Docker::{
                         , image =
                             (../Constants/ContainerImages.dhall).toolchainBase

buildkite/src/Jobs/Lint/Dhall.dhall

@@ -12,8 +12,6 @@ let Command = ../../Command/Base.dhall
 
 let Docker = ../../Command/Docker/Type.dhall
 
-let Size = ../../Command/Size.dhall
-
 in  Pipeline.build
       Pipeline.Config::{
       , spec = JobSpec::{
@@ -33,7 +31,6 @@ in  Pipeline.build
             , commands = [ Cmd.run "cd buildkite && make check_syntax" ]
             , label = "Dhall: syntax"
             , key = "check-dhall-syntax"
-            , target = Size.Small
             , docker = Some Docker::{
               , image = (../../Constants/ContainerImages.dhall).toolchainBase
               }
@@ -43,7 +40,6 @@ in  Pipeline.build
             , commands = [ Cmd.run "cd buildkite && make check_lint" ]
             , label = "Dhall: lint"
             , key = "check-dhall-lint"
-            , target = Size.Small
             , docker = Some Docker::{
               , image = (../../Constants/ContainerImages.dhall).toolchainBase
               }
@@ -53,7 +49,6 @@ in  Pipeline.build
             , commands = [ Cmd.run "cd buildkite && make check_format" ]
             , label = "Dhall: format"
             , key = "check-dhall-format"
-            , target = Size.Small
             , docker = Some Docker::{
               , image = (../../Constants/ContainerImages.dhall).toolchainBase
               }

buildkite/src/Jobs/Lint/Fast.dhall

@@ -12,8 +12,6 @@ let Command = ../../Command/Base.dhall
 
 let Docker = ../../Command/Docker/Type.dhall
 
-let Size = ../../Command/Size.dhall
-
 let commands =
       [ Cmd.run "./scripts/lint_codeowners.sh"
       , Cmd.run "./scripts/lint_rfcs.sh"
@@ -45,7 +43,6 @@ in  Pipeline.build
             , label =
                 "Fast lint steps; CODEOWNERs, RFCs, Check Snarky & Proof-Systems submodules, Preprocessor Deps"
             , key = "lint"
-            , target = Size.Small
             , docker = Some Docker::{
               , image = (../../Constants/ContainerImages.dhall).toolchainBase
               }

buildkite/src/Jobs/Lint/HelmChart.dhall

@@ -12,8 +12,6 @@ let Command = ../../Command/Base.dhall
 
 let Docker = ../../Command/Docker/Type.dhall
 
-let Size = ../../Command/Size.dhall
-
 in  Pipeline.build
       Pipeline.Config::{
       , spec = JobSpec::{
@@ -39,7 +37,6 @@ in  Pipeline.build
               [ Cmd.run "HELM_LINT=true buildkite/scripts/helm-ci.sh" ]
             , label = "Helm chart lint steps"
             , key = "lint-helm-chart"
-            , target = Size.Small
             , docker = None Docker.Type
             }
         ]