[projmgr] Add --frozen-packs argument to csolution

The --frozen-packs option is only applicable for the following commands:
- convert
- update-rte

If cbuild-pack.yml needs to be created or updated, then --frozen-pack will
treat this as an error and fail the command.

Contributed by STMicroelectronics

Signed-off-by: Torbjörn SVENSSON <[email protected]>

tools/projmgr/include/ProjMgr.h

@@ -130,6 +130,7 @@ class ProjMgr {
   bool m_ymlOrder;
   bool m_contextSet;
   bool m_relativePaths;
+  bool m_frozenPacks;
   GroupNode m_files;
   std::vector<ContextItem*> m_processedContexts;
 

tools/projmgr/include/ProjMgrParser.h

@@ -499,8 +499,9 @@ class ProjMgrParser {
    * @brief parse csolution
    * @param checkSchema false to skip schema validation
    * @param input csolution.yml file
+   * @param frozenPacks false to allow missing cbuild-packs.yml file
   */
-  bool ParseCsolution(const std::string& input, bool checkSchema);
+  bool ParseCsolution(const std::string& input, bool checkSchema, bool frozenPacks);
 
   /**
    * @brief parse clayer

tools/projmgr/include/ProjMgrYamlEmitter.h

@@ -68,9 +68,10 @@ class ProjMgrYamlEmitter {
    * @brief generate cbuild pack file
    * @param contexts vector with pointers to contexts
    * @param keepExistingPackContent if true, all entries from existing cbuild-pack should be preserved
+   * @param cbuildPackFrozen if true, reject updates to cbuild pack file
    * @return true if executed successfully
   */
-  static bool GenerateCbuildPack(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, bool keepExistingPackContent);
+  static bool GenerateCbuildPack(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, bool keepExistingPackContent, bool cbuildPackFrozen);
 };
 
 #endif  // PROJMGRYAMLEMITTER_H

tools/projmgr/include/ProjMgrYamlParser.h

@@ -176,9 +176,10 @@ class ProjMgrYamlParser {
    * @param input csolution.yml file
    * @param reference to store parsed csolution item
    * @param checkSchema false to skip schema validation
+   * @param frozenPacks false to allow missing cbuild-packs.yml file
   */
   bool ParseCsolution(const std::string& input, CsolutionItem& csolution,
-    bool checkSchema);
+    bool checkSchema, bool frozenPacks);
 
   /**
    * @brief parse cproject

tools/projmgr/src/ProjMgr.cpp

@@ -69,7 +69,8 @@ ProjMgr::ProjMgr() :
   m_dryRun(false),
   m_ymlOrder(false),
   m_contextSet(false),
-  m_relativePaths(false)
+  m_relativePaths(false),
+  m_frozenPacks(false)
 {
 }
 
@@ -151,12 +152,13 @@ int ProjMgr::ParseCommandLine(int argc, char** argv) {
   cxxopts::Option ymlOrder("yml-order", "Preserve order as specified in input yml", cxxopts::value<bool>()->default_value("false"));
   cxxopts::Option contextSet("S,context-set", "Use context set", cxxopts::value<bool>()->default_value("false"));
   cxxopts::Option relativePaths("R,relative-paths", "Output paths relative to project or to CMSIS_PACK_ROOT", cxxopts::value<bool>()->default_value("false"));
+  cxxopts::Option frozenPacks("frozen-packs", "The list of packs from cbuild-pack.yml is frozen and raises error if not up-to-date", cxxopts::value<bool>()->default_value("false"));
 
   // command options dictionary
   map<string, std::pair<bool, vector<cxxopts::Option>>> optionsDict = {
     // command, optional args, options
-    {"update-rte",        { false, {context, contextSet, debug, load, schemaCheck, toolchain, verbose}}},
-    {"convert",           { false, {context, contextSet, debug, exportSuffix, load, schemaCheck, noUpdateRte, output, toolchain, verbose}}},
+    {"update-rte",        { false, {context, contextSet, debug, load, schemaCheck, toolchain, verbose, frozenPacks}}},
+    {"convert",           { false, {context, contextSet, debug, exportSuffix, load, schemaCheck, noUpdateRte, output, toolchain, verbose, frozenPacks}}},
     {"run",               { false, {context, debug, generator, load, schemaCheck, verbose, dryRun}}},
     {"list packs",        { true,  {context, debug, filter, load, missing, schemaCheck, toolchain, verbose, relativePaths}}},
     {"list boards",       { true,  {context, debug, filter, load, schemaCheck, toolchain, verbose}}},
@@ -176,7 +178,7 @@ int ProjMgr::ParseCommandLine(int argc, char** argv) {
       {"positional", "", cxxopts::value<vector<string>>()},
       solution, context, contextSet, filter, generator,
       load, clayerSearchPath, missing, schemaCheck, noUpdateRte, output,
-      help, version, verbose, debug, dryRun, exportSuffix, toolchain, ymlOrder, relativePaths
+      help, version, verbose, debug, dryRun, exportSuffix, toolchain, ymlOrder, relativePaths, frozenPacks
     });
     options.parse_positional({ "positional" });
 
@@ -196,6 +198,7 @@ int ProjMgr::ParseCommandLine(int argc, char** argv) {
     m_contextSet = parseResult.count("context-set");
     m_relativePaths = parseResult.count("relative-paths");
     m_worker.SetPrintRelativePaths(m_relativePaths);
+    m_frozenPacks = parseResult.count("frozen-packs");
 
     vector<string> positionalArguments;
     if (parseResult.count("positional")) {
@@ -398,7 +401,7 @@ bool ProjMgr::SetLoadPacksPolicy(void) {
 bool ProjMgr::PopulateContexts(void) {
   if (!m_csolutionFile.empty()) {
     // Parse csolution
-    if (!m_parser.ParseCsolution(m_csolutionFile, m_checkSchema)) {
+    if (!m_parser.ParseCsolution(m_csolutionFile, m_checkSchema, m_frozenPacks)) {
       return false;
     }
     // Parse cdefault
@@ -552,12 +555,13 @@ bool ProjMgr::RunConfigure(bool printConfig) {
     if (!m_emitter.GenerateCbuild(contextItem)) {
       return false;
     }
-
   }
 
   // Generate cbuild-pack file
   const bool isUsingContexts = m_contextSet || m_context.size() != 0;
-  m_emitter.GenerateCbuildPack(m_parser, m_processedContexts, isUsingContexts);
+  if (!m_emitter.GenerateCbuildPack(m_parser, m_processedContexts, isUsingContexts, m_frozenPacks)) {
+    return false;
+  }
 
   // Update the RTE files
   if (m_updateRteFiles) {

tools/projmgr/src/ProjMgrParser.cpp

@@ -28,9 +28,9 @@ bool ProjMgrParser::ParseCdefault(const string& input, bool checkSchema) {
   return ProjMgrYamlParser().ParseCdefault(input, m_cdefault, checkSchema);
 }
 
-bool ProjMgrParser::ParseCsolution(const string& input, bool checkSchema) {
+bool ProjMgrParser::ParseCsolution(const string& input, bool checkSchema, bool frozenPacks) {
   // Parse solution file
-  return ProjMgrYamlParser().ParseCsolution(input, m_csolution, checkSchema);
+  return ProjMgrYamlParser().ParseCsolution(input, m_csolution, checkSchema, frozenPacks);
 }
 
 bool ProjMgrParser::ParseCproject(const string& input, bool checkSchema, bool single) {

tools/projmgr/src/ProjMgrYamlEmitter.cpp

@@ -35,7 +35,7 @@ class ProjMgrYamlBase {
   const string FormatPath(const string& original, const string& directory);
   bool CompareFile(const string& filename, const YAML::Node& rootNode);
   bool CompareNodes(const YAML::Node& lhs, const YAML::Node& rhs);
-  bool WriteFile(YAML::Node& rootNode, const std::string& filename);
+  bool WriteFile(YAML::Node& rootNode, const std::string& filename, bool allowUpdate = true);
   const bool m_useAbsolutePaths;
 };
 
@@ -705,7 +705,7 @@ bool ProjMgrYamlBase::CompareNodes(const YAML::Node& lhs, const YAML::Node& rhs)
   return (lhsData == rhsData) ? true : false;
 }
 
-bool ProjMgrYamlBase::WriteFile(YAML::Node& rootNode, const std::string& filename) {
+bool ProjMgrYamlBase::WriteFile(YAML::Node& rootNode, const std::string& filename, bool allowUpdate) {
   // Compare yaml contents
   if (RteFsUtils::IsDirectory(filename)) {
     ProjMgrLogger::Error(filename, "file cannot be written");
@@ -721,6 +721,10 @@ bool ProjMgrYamlBase::WriteFile(YAML::Node& rootNode, const std::string& filenam
     }
   }
   else if (!CompareFile(filename, rootNode)) {
+    if (!allowUpdate) {
+      ProjMgrLogger::Error(filename, "file not allowed to be updated");
+      return false;
+    }
     if (!RteFsUtils::MakeSureFilePath(filename)) {
       ProjMgrLogger::Error(filename, "destination directory can not be created");
       return false;
@@ -829,12 +833,12 @@ bool ProjMgrYamlEmitter::GenerateCbuildGenIndex(ProjMgrParser& parser, const vec
   return cbuild.WriteFile(rootNode, filename);
 }
 
-bool ProjMgrYamlEmitter::GenerateCbuildPack(ProjMgrParser& parser, const vector<ContextItem*> contexts, bool keepExistingPackContent) {
+bool ProjMgrYamlEmitter::GenerateCbuildPack(ProjMgrParser& parser, const vector<ContextItem*> contexts, bool keepExistingPackContent, bool cbuildPackFrozen) {
   // generate cbuild-pack.yml
   const string& filename = parser.GetCsolution().directory + "/" + parser.GetCsolution().name + ".cbuild-pack.yml";
 
   YAML::Node rootNode;
   ProjMgrYamlCbuildPack cbuildPack(rootNode[YAML_CBUILD_PACK], contexts, parser, keepExistingPackContent);
 
-  return cbuildPack.WriteFile(rootNode, filename);
+  return cbuildPack.WriteFile(rootNode, filename, !cbuildPackFrozen);
 }

tools/projmgr/src/ProjMgrYamlParser.cpp

@@ -52,13 +52,16 @@ bool ProjMgrYamlParser::ParseCdefault(const string& input,
 }
 
 bool ProjMgrYamlParser::ParseCsolution(const string& input,
-  CsolutionItem& csolution, bool checkSchema) {
+  CsolutionItem& csolution, bool checkSchema, bool frozenPacks) {
 
   string cbuildPackFile = RteUtils::RemoveSuffixByString(input, ".csolution.yml") + ".cbuild-pack.yml";
   if (fs::exists(cbuildPackFile)) {
     if (!ParseCbuildPack(cbuildPackFile, csolution.cbuildPack, checkSchema)) {
       return false;
     }
+  } else if (frozenPacks) {
+    ProjMgrLogger::Error(cbuildPackFile, "file is missing and required due to use of --frozen-packs option");
+    return false;
   }
 
   try {

tools/projmgr/test/data/TestSolution/PackLocking/cbuild_pack_frozen.cbuild-pack.yml

@@ -0,0 +1,5 @@
+cbuild-pack:
+  resolved-packs:
+    - resolved-pack: ARM::[email protected]
+      selected-by:
+        - ARM::RteTest_DFP

tools/projmgr/test/data/TestSolution/PackLocking/cbuild_pack_frozen.csolution.yml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Open-CMSIS-Pack/devtools/main/tools/projmgr/schemas/csolution.schema.json
+
+solution:
+  target-types:
+    - type: CM3
+      device: RteTest_ARMCM3
+  packs:
+    - pack: ARM::[email protected]
+  projects:
+    - project: ./project_with_dfp_components.cproject.yml

tools/projmgr/test/data/TestSolution/PackLocking/cbuild_pack_frozen_no_pack_file.csolution.yml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Open-CMSIS-Pack/devtools/main/tools/projmgr/schemas/csolution.schema.json
+
+solution:
+  target-types:
+    - type: CM0
+      device: RteTest_ARMCM0
+  packs:
+    - pack: ARM::[email protected]
+  projects:
+    - project: ./project_with_dfp_components.cproject.yml

tools/projmgr/test/data/TestSolution/PackLocking/ref/cbuild_pack_frozen.cbuild-pack.yml

@@ -0,0 +1,5 @@
+cbuild-pack:
+  resolved-packs:
+    - resolved-pack: ARM::[email protected]
+      selected-by:
+        - ARM::[email protected]

tools/projmgr/test/src/ProjMgrUnitTests.cpp

@@ -849,7 +849,6 @@ TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackCleanup) {
 
 TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackNoPackList) {
   char* argv[6];
-  string buf1, buf2;
 
   // convert --solution solution.yml
   const string csolution = testinput_folder + "/TestSolution/PackLocking/project_pack_lock_no_pack_list.csolution.yml";
@@ -871,6 +870,91 @@ TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackNoPackList) {
   ProjMgrTestEnv::CompareFile(expectedCbuildPack, cbuildPack);
 }
 
+TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackFrozen) {
+  char* argv[7];
+  StdStreamRedirect streamRedirect;
+
+  // convert --solution solution.yml
+  const string csolution = testinput_folder + "/TestSolution/PackLocking/cbuild_pack_frozen.csolution.yml";
+  const string cbuildPack = testinput_folder + "/TestSolution/PackLocking/cbuild_pack_frozen.cbuild-pack.yml";
+  const string expectedCbuildPackRef = testinput_folder + "/TestSolution/PackLocking/ref/cbuild_pack_frozen.cbuild-pack.yml";
+  const string rtePath = testinput_folder + "/TestSolution/PackLocking/RTE/";
+  const string expectedCbuildPack = RteFsUtils::BackupFile(cbuildPack);
+  const string output = testoutput_folder + "/testpacklock";
+  argv[1] = (char*)"convert";
+  argv[2] = (char*)"--solution";
+  argv[3] = (char*)csolution.c_str();
+  argv[4] = (char*)"-o";
+  argv[5] = (char*)output.c_str();
+  argv[6] = (char*)"--frozen-packs";
+
+  // Ensure clean state when starting test
+  ASSERT_TRUE(RteFsUtils::RemoveDir(rtePath));
+
+  // 1st run to verify that the cbuild-pack.yml content is stable
+  EXPECT_NE(0, RunProjMgr(7, argv, 0));
+  EXPECT_NE(streamRedirect.GetErrorString().find(cbuildPack + " - error csolution: file not allowed to be updated"), string::npos);
+  ProjMgrTestEnv::CompareFile(expectedCbuildPack, cbuildPack);
+  EXPECT_FALSE(RteFsUtils::Exists(rtePath));
+
+  // 2nd run to verify that the cbuild-pack.yml content is stable
+  streamRedirect.ClearStringStreams();
+  EXPECT_NE(0, RunProjMgr(7, argv, 0));
+  EXPECT_NE(streamRedirect.GetErrorString().find(cbuildPack + " - error csolution: file not allowed to be updated"), string::npos);
+  ProjMgrTestEnv::CompareFile(expectedCbuildPack, cbuildPack);
+  EXPECT_FALSE(RteFsUtils::Exists(rtePath));
+
+  // 3rd run without --frozen-packs to verify that the list can be updated
+  streamRedirect.ClearStringStreams();
+  EXPECT_EQ(0, RunProjMgr(6, argv, 0));
+  EXPECT_NE(streamRedirect.GetOutString().find(cbuildPack + " - info csolution: file generated successfully"), string::npos);
+  ProjMgrTestEnv::CompareFile(expectedCbuildPackRef, cbuildPack);
+  EXPECT_TRUE(RteFsUtils::Exists(rtePath));
+
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/_CM3/RTE_Components.h"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/gcc_arm.ld"));
+  EXPECT_FALSE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]")); // From Arm::[email protected]
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/startup_ARMCM3.c"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/system_ARMCM3.c"));
+  EXPECT_FALSE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]")); // From Arm::[email protected]
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]"));
+
+  // 4th run with --frozen-packs to verify that RTE directory can be generated
+  ASSERT_TRUE(RteFsUtils::RemoveDir(rtePath));
+  streamRedirect.ClearStringStreams();
+  EXPECT_EQ(0, RunProjMgr(7, argv, 0));
+  EXPECT_NE(streamRedirect.GetOutString().find(cbuildPack + " - info csolution: file is already up-to-date"), string::npos);
+  ProjMgrTestEnv::CompareFile(expectedCbuildPackRef, cbuildPack);
+
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/_CM3/RTE_Components.h"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/gcc_arm.ld"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/startup_ARMCM3.c"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/system_ARMCM3.c"));
+  EXPECT_TRUE(RteFsUtils::Exists(testinput_folder + "/TestSolution/PackLocking/RTE/Device/RteTest_ARMCM3/[email protected]"));
+
+  RteFsUtils::RemoveFile(expectedCbuildPack);
+}
+
+TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackFrozenNoPackFile) {
+  char* argv[7];
+
+  // convert --solution solution.yml
+  const string csolution = testinput_folder + "/TestSolution/PackLocking/cbuild_pack_frozen_no_pack_file.csolution.yml";
+  const string output = testoutput_folder + "/testpacklock";
+  argv[1] = (char*)"convert";
+  argv[2] = (char*)"--solution";
+  argv[3] = (char*)csolution.c_str();
+  argv[4] = (char*)"-o";
+  argv[5] = (char*)output.c_str();
+  argv[6] = (char*)"--frozen-packs";
+
+  EXPECT_NE(0, RunProjMgr(7, argv, 0));
+}
+
 TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackReselectSelectedBy) {
   char* argv[6];
 
@@ -2074,7 +2158,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts) {
   const string& dirInput = testinput_folder + "/TestSolution/";
   const string& filenameInput = dirInput + "test.csolution.yml";
   error_code ec;
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   for (const auto& cproject : m_parser.GetCsolution().cprojects) {
     string const& cprojectFile = fs::canonical(dirInput + cproject, ec).generic_string();
     EXPECT_TRUE(m_parser.ParseCproject(cprojectFile, false, false));
@@ -2098,7 +2182,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts_Ordered) {
   const string& dirInput = testinput_folder + "/TestSolution/";
   const string& filenameInput = dirInput + "test_ordered.csolution.yml";
   error_code ec;
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   for (const auto& cproject : m_parser.GetCsolution().cprojects) {
     string const& cprojectFile = fs::canonical(dirInput + cproject, ec).generic_string();
     EXPECT_TRUE(m_parser.ParseCproject(cprojectFile, false, false));
@@ -2123,7 +2207,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts_Without_BuildTypes) {
   const string& dirInput = testinput_folder + "/TestSolution/";
   const string& filenameInput = dirInput + "test_no_buildtypes.csolution.yml";
   error_code ec;
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   for (const auto& cproject : m_parser.GetCsolution().cprojects) {
     string const& cprojectFile = fs::canonical(dirInput + cproject, ec).generic_string();
     EXPECT_TRUE(m_parser.ParseCproject(cprojectFile, false, false));
@@ -2140,7 +2224,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts_Without_BuildTypes) {
 TEST_F(ProjMgrUnitTests, AddContextFailed) {
   ContextDesc descriptor;
   const string& filenameInput = testinput_folder + "/TestSolution/test_missing_project.csolution.yml";
-  EXPECT_FALSE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_FALSE(m_parser.ParseCsolution(filenameInput, false, false));
   EXPECT_FALSE(m_worker.AddContexts(m_parser, descriptor, filenameInput));
 }
 
@@ -3540,7 +3624,7 @@ TEST_F(ProjMgrUnitTests, RunProjMgr_PreInclude) {
 
 TEST_F(ProjMgrUnitTests, RunCheckForContext) {
   const string& filenameInput = testinput_folder + "/TestSolution/contexts.csolution.yml";
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   const CsolutionItem csolutionItem = m_parser.GetCsolution();
   const auto& contexts = csolutionItem.contexts;
   const string& cproject = "contexts.cproject.yml";