[projmgr] Add --frozen-packs argument to csolution

The --frozen-packs option is only applicable for the following commands:
- convert
- run
- update-rte

This current implementation will do the command and then check if it's
supposed to be failed or not. This does support the CI use case, but
the project will be modified regardless of the --frozen-packs flag.

Contributed by STMicroelectronics

Signed-off-by: Torbjörn SVENSSON <[email protected]>

tools/projmgr/include/ProjMgr.h

@@ -113,6 +113,7 @@ class ProjMgr {
   bool m_ymlOrder;
   bool m_contextSet;
   bool m_relativePaths;
+  bool m_frozenPacks;
   GroupNode m_files;
   std::vector<ContextItem*> m_processedContexts;
 

tools/projmgr/include/ProjMgrParser.h

@@ -499,8 +499,9 @@ class ProjMgrParser {
    * @brief parse csolution
    * @param checkSchema false to skip schema validation
    * @param input csolution.yml file
+   * @param frozenPacks false to allow missing cbuild-packs.yml file
   */
-  bool ParseCsolution(const std::string& input, bool checkSchema);
+  bool ParseCsolution(const std::string& input, bool checkSchema, bool frozenPacks);
 
   /**
    * @brief parse clayer

tools/projmgr/include/ProjMgrYamlEmitter.h

@@ -68,9 +68,10 @@ class ProjMgrYamlEmitter {
    * @brief generate cbuild pack file
    * @param contexts vector with pointers to contexts
    * @param keepExistingPackContent if true, all entries from existing cbuild-pack should be preserved
+   * @param cbuildPackFrozen if true, reject updates to cbuild pack file
    * @return true if executed successfully
   */
-  static bool GenerateCbuildPack(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, bool keepExistingPackContent);
+  static bool GenerateCbuildPack(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, bool keepExistingPackContent, bool cbuildPackFrozen);
 };
 
 #endif  // PROJMGRYAMLEMITTER_H

tools/projmgr/include/ProjMgrYamlParser.h

@@ -176,9 +176,10 @@ class ProjMgrYamlParser {
    * @param input csolution.yml file
    * @param reference to store parsed csolution item
    * @param checkSchema false to skip schema validation
+   * @param frozenPacks false to allow missing cbuild-packs.yml file
   */
   bool ParseCsolution(const std::string& input, CsolutionItem& csolution,
-    bool checkSchema);
+    bool checkSchema, bool frozenPacks);
 
   /**
    * @brief parse cproject

tools/projmgr/src/ProjMgr.cpp

@@ -69,7 +69,8 @@ ProjMgr::ProjMgr() :
   m_dryRun(false),
   m_ymlOrder(false),
   m_contextSet(false),
-  m_relativePaths(false)
+  m_relativePaths(false),
+  m_frozenPacks(false)
 {
 }
 
@@ -152,13 +153,14 @@ int ProjMgr::RunProjMgr(int argc, char **argv, char** envp) {
   cxxopts::Option ymlOrder("yml-order", "Preserve order as specified in input yml", cxxopts::value<bool>()->default_value("false"));
   cxxopts::Option contextSet("S,context-set", "Use context set", cxxopts::value<bool>()->default_value("false"));
   cxxopts::Option relativePaths("R,relative-paths", "Output paths relative to project or to CMSIS_PACK_ROOT", cxxopts::value<bool>()->default_value("false"));
+  cxxopts::Option frozenPacks("frozen-packs", "The list of packs from cbuild-pack.yml is frozen and raises error if not up-to-date", cxxopts::value<bool>()->default_value("false"));
 
   // command options dictionary
   map<string, std::pair<bool, vector<cxxopts::Option>>> optionsDict = {
     // command, optional args, options
-    {"update-rte",        { false, {context, contextSet, debug, load, schemaCheck, toolchain, verbose}}},
-    {"convert",           { false, {context, contextSet, debug, exportSuffix, load, schemaCheck, noUpdateRte, output, toolchain, verbose}}},
-    {"run",               { false, {context, debug, generator, load, schemaCheck, verbose, dryRun}}},
+    {"update-rte",        { false, {context, contextSet, debug, load, schemaCheck, toolchain, verbose, frozenPacks}}},
+    {"convert",           { false, {context, contextSet, debug, exportSuffix, load, schemaCheck, noUpdateRte, output, toolchain, verbose, frozenPacks}}},
+    {"run",               { false, {context, debug, generator, load, schemaCheck, verbose, dryRun, frozenPacks}}},
     {"list packs",        { true,  {context, debug, filter, load, missing, schemaCheck, toolchain, verbose, relativePaths}}},
     {"list boards",       { true,  {context, debug, filter, load, schemaCheck, toolchain, verbose}}},
     {"list devices",      { true,  {context, debug, filter, load, schemaCheck, toolchain, verbose}}},
@@ -177,7 +179,7 @@ int ProjMgr::RunProjMgr(int argc, char **argv, char** envp) {
       {"positional", "", cxxopts::value<vector<string>>()},
       solution, context, contextSet, filter, generator,
       load, clayerSearchPath, missing, schemaCheck, noUpdateRte, output,
-      help, version, verbose, debug, dryRun, exportSuffix, toolchain, ymlOrder, relativePaths
+      help, version, verbose, debug, dryRun, exportSuffix, toolchain, ymlOrder, relativePaths, frozenPacks
     });
     options.parse_positional({ "positional" });
 
@@ -197,6 +199,7 @@ int ProjMgr::RunProjMgr(int argc, char **argv, char** envp) {
     manager.m_contextSet = parseResult.count("context-set");
     manager.m_relativePaths = parseResult.count("relative-paths");
     manager.m_worker.SetPrintRelativePaths(manager.m_relativePaths);
+    manager.m_frozenPacks = parseResult.count("frozen-packs");
 
     vector<string> positionalArguments;
     if (parseResult.count("positional")) {
@@ -382,7 +385,7 @@ bool ProjMgr::SetLoadPacksPolicy(void) {
 bool ProjMgr::PopulateContexts(void) {
   if (!m_csolutionFile.empty()) {
     // Parse csolution
-    if (!m_parser.ParseCsolution(m_csolutionFile, m_checkSchema)) {
+    if (!m_parser.ParseCsolution(m_csolutionFile, m_checkSchema, m_frozenPacks)) {
       return false;
     }
     // Parse cdefault
@@ -536,12 +539,13 @@ bool ProjMgr::RunConfigure(bool printConfig) {
     if (!m_emitter.GenerateCbuild(contextItem)) {
       return false;
     }
-
   }
 
   // Generate cbuild-pack file
   const bool isUsingContexts = m_contextSet || m_context.size() != 0;
-  m_emitter.GenerateCbuildPack(m_parser, m_processedContexts, isUsingContexts);
+  if (!m_emitter.GenerateCbuildPack(m_parser, m_processedContexts, isUsingContexts, m_frozenPacks)) {
+    return false;
+  }
 
   return !error;
 }

tools/projmgr/src/ProjMgrParser.cpp

@@ -28,9 +28,9 @@ bool ProjMgrParser::ParseCdefault(const string& input, bool checkSchema) {
   return ProjMgrYamlParser().ParseCdefault(input, m_cdefault, checkSchema);
 }
 
-bool ProjMgrParser::ParseCsolution(const string& input, bool checkSchema) {
+bool ProjMgrParser::ParseCsolution(const string& input, bool checkSchema, bool frozenPacks) {
   // Parse solution file
-  return ProjMgrYamlParser().ParseCsolution(input, m_csolution, checkSchema);
+  return ProjMgrYamlParser().ParseCsolution(input, m_csolution, checkSchema, frozenPacks);
 }
 
 bool ProjMgrParser::ParseCproject(const string& input, bool checkSchema, bool single) {

tools/projmgr/src/ProjMgrYamlEmitter.cpp

@@ -35,7 +35,7 @@ class ProjMgrYamlBase {
   const string FormatPath(const string& original, const string& directory);
   bool CompareFile(const string& filename, const YAML::Node& rootNode);
   bool CompareNodes(const YAML::Node& lhs, const YAML::Node& rhs);
-  bool WriteFile(YAML::Node& rootNode, const std::string& filename);
+  bool WriteFile(YAML::Node& rootNode, const std::string& filename, bool allowUpdate = true);
   const bool m_useAbsolutePaths;
 };
 
@@ -705,7 +705,7 @@ bool ProjMgrYamlBase::CompareNodes(const YAML::Node& lhs, const YAML::Node& rhs)
   return (lhsData == rhsData) ? true : false;
 }
 
-bool ProjMgrYamlBase::WriteFile(YAML::Node& rootNode, const std::string& filename) {
+bool ProjMgrYamlBase::WriteFile(YAML::Node& rootNode, const std::string& filename, bool allowUpdate) {
   // Compare yaml contents
   if (RteFsUtils::IsDirectory(filename)) {
     ProjMgrLogger::Error(filename, "file cannot be written");
@@ -721,6 +721,10 @@ bool ProjMgrYamlBase::WriteFile(YAML::Node& rootNode, const std::string& filenam
     }
   }
   else if (!CompareFile(filename, rootNode)) {
+    if (!allowUpdate) {
+      ProjMgrLogger::Error(filename, "file not allowed to be updated");
+      return false;
+    }
     if (!RteFsUtils::MakeSureFilePath(filename)) {
       ProjMgrLogger::Error(filename, "destination directory can not be created");
       return false;
@@ -829,12 +833,12 @@ bool ProjMgrYamlEmitter::GenerateCbuildGenIndex(ProjMgrParser& parser, const vec
   return cbuild.WriteFile(rootNode, filename);
 }
 
-bool ProjMgrYamlEmitter::GenerateCbuildPack(ProjMgrParser& parser, const vector<ContextItem*> contexts, bool keepExistingPackContent) {
+bool ProjMgrYamlEmitter::GenerateCbuildPack(ProjMgrParser& parser, const vector<ContextItem*> contexts, bool keepExistingPackContent, bool cbuildPackFrozen) {
   // generate cbuild-pack.yml
   const string& filename = parser.GetCsolution().directory + "/" + parser.GetCsolution().name + ".cbuild-pack.yml";
 
   YAML::Node rootNode;
   ProjMgrYamlCbuildPack cbuildPack(rootNode[YAML_CBUILD_PACK], contexts, parser, keepExistingPackContent);
 
-  return cbuildPack.WriteFile(rootNode, filename);
+  return cbuildPack.WriteFile(rootNode, filename, !cbuildPackFrozen);
 }

tools/projmgr/src/ProjMgrYamlParser.cpp

@@ -52,13 +52,16 @@ bool ProjMgrYamlParser::ParseCdefault(const string& input,
 }
 
 bool ProjMgrYamlParser::ParseCsolution(const string& input,
-  CsolutionItem& csolution, bool checkSchema) {
+  CsolutionItem& csolution, bool checkSchema, bool frozenPacks) {
 
   string cbuildPackFile = RteUtils::RemoveSuffixByString(input, ".csolution.yml") + ".cbuild-pack.yml";
   if (fs::exists(cbuildPackFile)) {
     if (!ParseCbuildPack(cbuildPackFile, csolution.cbuildPack, checkSchema)) {
       return false;
     }
+  } else if (frozenPacks) {
+    ProjMgrLogger::Error(cbuildPackFile, "file is missing and required due to use of --frozen-packs option");
+    return false;
   }
 
   try {

tools/projmgr/test/data/TestSolution/PackLocking/cbuild_pack_frozen.cbuild-pack.yml

@@ -0,0 +1,5 @@
+cbuild-pack:
+  resolved-packs:
+    - resolved-pack: ARM::[email protected]
+      selected-by:
+        - ARM::RteTest_DFP

tools/projmgr/test/data/TestSolution/PackLocking/cbuild_pack_frozen.csolution.yml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Open-CMSIS-Pack/devtools/main/tools/projmgr/schemas/csolution.schema.json
+
+solution:
+  target-types:
+    - type: CM0
+      device: RteTest_ARMCM0
+  packs:
+    - pack: ARM::[email protected]
+  projects:
+    - project: ./project_with_dfp_components.cproject.yml

tools/projmgr/test/data/TestSolution/PackLocking/cbuild_pack_frozen_no_pack_file.csolution.yml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Open-CMSIS-Pack/devtools/main/tools/projmgr/schemas/csolution.schema.json
+
+solution:
+  target-types:
+    - type: CM0
+      device: RteTest_ARMCM0
+  packs:
+    - pack: ARM::[email protected]
+  projects:
+    - project: ./project_with_dfp_components.cproject.yml

tools/projmgr/test/src/ProjMgrUnitTests.cpp

@@ -849,7 +849,6 @@ TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackCleanup) {
 
 TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackNoPackList) {
   char* argv[6];
-  string buf1, buf2;
 
   // convert --solution solution.yml
   const string csolution = testinput_folder + "/TestSolution/PackLocking/project_pack_lock_no_pack_list.csolution.yml";
@@ -871,6 +870,61 @@ TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackNoPackList) {
   ProjMgrTestEnv::CompareFile(expectedCbuildPack, cbuildPack);
 }
 
+TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackFrozen) {
+  char* argv[7];
+  StdStreamRedirect streamRedirect;
+  string buf;
+
+  // convert --solution solution.yml
+  const string csolution = testinput_folder + "/TestSolution/PackLocking/cbuild_pack_frozen.csolution.yml";
+  const string cbuildPack = testinput_folder + "/TestSolution/PackLocking/cbuild_pack_frozen.cbuild-pack.yml";
+  const string expectedCbuildPack = RteFsUtils::BackupFile(cbuildPack);
+  const string output = testoutput_folder + "/testpacklock";
+  argv[1] = (char*)"convert";
+  argv[2] = (char*)"--solution";
+  argv[3] = (char*)csolution.c_str();
+  argv[4] = (char*)"-o";
+  argv[5] = (char*)output.c_str();
+  argv[6] = (char*)"--frozen-packs";
+
+  EXPECT_NE(0, RunProjMgr(7, argv, 0));
+  EXPECT_NE(streamRedirect.GetErrorString().find(cbuildPack + " - error csolution: file not allowed to be updated"), string::npos);
+  ProjMgrTestEnv::CompareFile(expectedCbuildPack, cbuildPack);
+
+  // 2nd run to verify that the cbuild-pack.yml content is stable
+  streamRedirect.ClearStringStreams();
+  EXPECT_NE(0, RunProjMgr(7, argv, 0));
+  EXPECT_NE(streamRedirect.GetErrorString().find(cbuildPack + " - error csolution: file not allowed to be updated"), string::npos);
+  ProjMgrTestEnv::CompareFile(expectedCbuildPack, cbuildPack);
+
+  // 3rd run without --frozen-packs to verify that the list can be updated
+  streamRedirect.ClearStringStreams();
+  EXPECT_EQ(0, RunProjMgr(6, argv, 0));
+  EXPECT_EQ(streamRedirect.GetErrorString().find(cbuildPack + " - error csolution: file not allowed to be updated"), string::npos);
+  EXPECT_TRUE(RteFsUtils::Exists(cbuildPack));
+  EXPECT_TRUE(RteFsUtils::ReadFile(cbuildPack, buf));
+  EXPECT_TRUE(buf.find("- resolved-pack: ARM::[email protected]") != string::npos);
+  EXPECT_TRUE(buf.find("- ARM::[email protected]") != string::npos);
+
+  RteFsUtils::RemoveFile(expectedCbuildPack);
+}
+
+TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackFrozenNoPackFile) {
+  char* argv[7];
+
+  // convert --solution solution.yml
+  const string csolution = testinput_folder + "/TestSolution/PackLocking/cbuild_pack_frozen_no_pack_file.csolution.yml";
+  const string output = testoutput_folder + "/testpacklock";
+  argv[1] = (char*)"convert";
+  argv[2] = (char*)"--solution";
+  argv[3] = (char*)csolution.c_str();
+  argv[4] = (char*)"-o";
+  argv[5] = (char*)output.c_str();
+  argv[6] = (char*)"--frozen-packs";
+
+  EXPECT_NE(0, RunProjMgr(7, argv, 0));
+}
+
 TEST_F(ProjMgrUnitTests, RunProjMgrSolution_LockPackReselectSelectedBy) {
   char* argv[6];
 
@@ -2074,7 +2128,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts) {
   const string& dirInput = testinput_folder + "/TestSolution/";
   const string& filenameInput = dirInput + "test.csolution.yml";
   error_code ec;
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   for (const auto& cproject : m_parser.GetCsolution().cprojects) {
     string const& cprojectFile = fs::canonical(dirInput + cproject, ec).generic_string();
     EXPECT_TRUE(m_parser.ParseCproject(cprojectFile, false, false));
@@ -2098,7 +2152,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts_Ordered) {
   const string& dirInput = testinput_folder + "/TestSolution/";
   const string& filenameInput = dirInput + "test_ordered.csolution.yml";
   error_code ec;
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   for (const auto& cproject : m_parser.GetCsolution().cprojects) {
     string const& cprojectFile = fs::canonical(dirInput + cproject, ec).generic_string();
     EXPECT_TRUE(m_parser.ParseCproject(cprojectFile, false, false));
@@ -2123,7 +2177,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts_Without_BuildTypes) {
   const string& dirInput = testinput_folder + "/TestSolution/";
   const string& filenameInput = dirInput + "test_no_buildtypes.csolution.yml";
   error_code ec;
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   for (const auto& cproject : m_parser.GetCsolution().cprojects) {
     string const& cprojectFile = fs::canonical(dirInput + cproject, ec).generic_string();
     EXPECT_TRUE(m_parser.ParseCproject(cprojectFile, false, false));
@@ -2140,7 +2194,7 @@ TEST_F(ProjMgrUnitTests, RunListContexts_Without_BuildTypes) {
 TEST_F(ProjMgrUnitTests, AddContextFailed) {
   ContextDesc descriptor;
   const string& filenameInput = testinput_folder + "/TestSolution/test_missing_project.csolution.yml";
-  EXPECT_FALSE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_FALSE(m_parser.ParseCsolution(filenameInput, false, false));
   EXPECT_FALSE(m_worker.AddContexts(m_parser, descriptor, filenameInput));
 }
 
@@ -3540,7 +3594,7 @@ TEST_F(ProjMgrUnitTests, RunProjMgr_PreInclude) {
 
 TEST_F(ProjMgrUnitTests, RunCheckForContext) {
   const string& filenameInput = testinput_folder + "/TestSolution/contexts.csolution.yml";
-  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false));
+  EXPECT_TRUE(m_parser.ParseCsolution(filenameInput, false, false));
   const CsolutionItem csolutionItem = m_parser.GetCsolution();
   const auto& contexts = csolutionItem.contexts;
   const string& cproject = "contexts.cproject.yml";