fix: fixing aggregation issues; marking unwrapped artifacts with skip…

… directive; adding tests

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/configuration/DirectoryScanAggregatorConfiguration.java

@@ -16,18 +16,13 @@
 package org.metaeffekt.core.inventory.processor.configuration;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.Validate;
-import org.apache.tools.ant.Project;
-import org.metaeffekt.core.inventory.InventoryUtils;
+import org.metaeffekt.core.inventory.InventoryMergeUtils;
 import org.metaeffekt.core.inventory.processor.filescan.ComponentPatternValidator;
-import org.metaeffekt.core.inventory.processor.model.Artifact;
-import org.metaeffekt.core.inventory.processor.model.ComponentPatternData;
-import org.metaeffekt.core.inventory.processor.model.FilePatternQualifierMapper;
-import org.metaeffekt.core.inventory.processor.model.Inventory;
+import org.metaeffekt.core.inventory.processor.model.*;
 import org.metaeffekt.core.inventory.processor.patterns.ComponentPatternProducer;
 import org.metaeffekt.core.inventory.processor.patterns.contributors.ContributorUtils;
-import org.metaeffekt.core.inventory.processor.reader.InventoryReader;
 import org.metaeffekt.core.util.ArchiveUtils;
+import org.metaeffekt.core.util.ArtifactUtils;
 import org.metaeffekt.core.util.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,14 +33,11 @@
 
 import static org.metaeffekt.core.inventory.processor.model.ComponentPatternData.Attribute.*;
 import static org.metaeffekt.core.inventory.processor.model.Constants.*;
-import static org.metaeffekt.core.inventory.processor.model.Constants.KEY_ARCHIVE_PATH;
 
 public class DirectoryScanAggregatorConfiguration {
 
     private static final Logger LOG = LoggerFactory.getLogger(DirectoryScanAggregatorConfiguration.class);
 
-    final private File referenceInventoryFile;
-
     final private Inventory referenceInventory;
 
     final private Inventory resultInventory;
@@ -57,19 +49,12 @@ public class DirectoryScanAggregatorConfiguration {
     public DirectoryScanAggregatorConfiguration(Inventory referenceInventory, Inventory resultInventory, File scanBaseDir) {
         this.scanBaseDir = scanBaseDir;
         this.referenceInventory = referenceInventory;
-        this.referenceInventoryFile = null;
         this.scanResultInventoryFile = null;
         this.resultInventory = resultInventory;
     }
 
     public List<FilePatternQualifierMapper> mapArtifactsToCoveredFiles() throws IOException {
 
-        // load reference inventory
-        final Inventory referenceInventory = loadReferenceInventory();
-
-        // load result inventory
-        final Inventory resultInventory = loadResultInventory();
-
         // initialize component pattern and file pattern map
         final Map<String, List<ComponentPatternData>> qualifierToComponentPatternMap = new HashMap<>();
         final List<FilePatternQualifierMapper> filePatternQualifierMapperList = new ArrayList<>();
@@ -115,11 +100,11 @@ public List<FilePatternQualifierMapper> mapArtifactsToCoveredFiles() throws IOEx
 
                 filePatternQualifierMapper.setFiles(componentPatternFiles);
             } else {
-                // handle artifacts that cannot be mapped to files; we need that the inventory is completely represented
-                // even in case no files are directly or indirectly associated
+                // handle artifacts that cannot be mapped to files by component patterns;
+                // we need that the inventory is completely represented even in case no files are directly or indirectly
+                // associated
                 filePatternQualifierMapper.setFileMap(Collections.emptyMap());
                 filePatternQualifierMapper.setFiles(Collections.emptyList());
-                // FIXME: add comprehensive test case for this
             }
 
             // add mapper
@@ -271,33 +256,6 @@ private Set<String> relativizePatterns(Set<String> patternSet) {
         return relativizedPatterns;
     }
 
-    private Inventory loadResultInventory() throws IOException {
-        if (resultInventory != null) {
-            return resultInventory;
-        } else {
-            final File scanResultInventoryFile = getResultInventoryFile();
-            FileUtils.validateExists(scanResultInventoryFile);
-            final Inventory inventory = new InventoryReader().readInventory(scanResultInventoryFile);
-            return inventory;
-        }
-    }
-
-    private Inventory loadReferenceInventory() throws IOException {
-        final Inventory referenceInventory;
-        if (referenceInventoryFile != null) {
-            FileUtils.validateExists(referenceInventoryFile);
-            if (referenceInventoryFile.isDirectory()) {
-                referenceInventory = InventoryUtils.readInventory(referenceInventoryFile, "*.xls");
-            } else {
-                referenceInventory = new InventoryReader().readInventory(referenceInventoryFile);
-            }
-        } else {
-            Validate.notNull(this.referenceInventory);
-            referenceInventory = this.referenceInventory;
-        }
-        return referenceInventory;
-    }
-
     private void contributeComponentPatterns(Inventory referenceInventory, Map<String, List<ComponentPatternData>> componentPatternMap) {
         for (ComponentPatternData cpd : referenceInventory.getComponentPatternData()) {
             final String key = deriveMapQualifier(cpd);
@@ -435,10 +393,6 @@ private void aggregateFilesForAllArtifacts(File scanBaseDir, List<FilePatternQua
 
         final Set<Artifact> coveredArtifacts = new HashSet<>();
 
-        // create an ant project
-        Project antProject = new Project();
-        antProject.init();
-
         for (FilePatternQualifierMapper mapper : filePatternQualifierMappers) {
             final File tmpFolder = FileUtils.initializeTmpFolder(targetDir);
 
@@ -498,12 +452,18 @@ private void aggregateFilesForAllArtifacts(File scanBaseDir, List<FilePatternQua
         // copy remaining artifacts not covered by component-patterns to aggregation dir
         for (Artifact artifact : resultInventory.getArtifacts()) {
             if (!coveredArtifacts.contains(artifact)) {
+
+                // evaluate directive
+                if (hasSkipAggregationDirective(artifact)) continue;
+
                 for (String project : artifact.getProjects()) {
                     File file = new File(scanBaseDir, project);
                     if (file.exists()) {
                         final String relativePath = FileUtils.asRelativePath(scanBaseDir, file);
                         try {
-                            FileUtils.copyFile(file, new File(targetDir, relativePath));
+                            final File targetFile = new File(targetDir, relativePath);
+                            FileUtils.copyFile(file, targetFile);
+                            artifact.set(KEY_ARCHIVE_PATH, targetFile.getAbsolutePath());
                         } catch (IOException e) {
                             LOG.warn("Cannot copy file [{}] to aggregation folder [{}]", file.getAbsolutePath(), targetDir.getAbsolutePath());
                         }
@@ -513,6 +473,11 @@ private void aggregateFilesForAllArtifacts(File scanBaseDir, List<FilePatternQua
         }
     }
 
+    private boolean hasSkipAggregationDirective(Artifact artifact) {
+        final String directive = artifact.get(KEY_AGGREGATE_DIRECTIVE);
+        return AGGREGATE_DIRECTIVE_SKIP.equalsIgnoreCase(directive);
+    }
+
     private File determineCommonRootDir(File scanBaseDir, List<File> files) {
         if (files == null || files.isEmpty()) return scanBaseDir;
 
@@ -547,4 +512,63 @@ private static String deriveQualifier(Artifact a) {
         return DirectoryScanAggregatorConfiguration.deriveMapQualifier(a.getComponent(), a.getVersion(), a.getId());
     }
 
+
+    public void contribute(File targetDir, Inventory aggregatedInventory) throws IOException {
+        aggregateFiles(targetDir);
+
+        final InventoryMergeUtils inventoryMergeUtils = new InventoryMergeUtils();
+        inventoryMergeUtils.setAddDefaultArtifactMergeAttributes(true);
+        inventoryMergeUtils.setAddDefaultArtifactExcludedAttributes(false);
+        inventoryMergeUtils.mergeInventories(Collections.singletonList(resultInventory), aggregatedInventory);
+
+        checkCompletenessOfArchivePath(aggregatedInventory);
+    }
+
+    /**
+     * There are multiple reasons for empty archive paths:
+     * <ul>
+     *     <li>
+     *          The relevant component pattern is not included in the reference inventory, This is a configuration issue.
+     *     </li>
+     *     <li>
+     *         There is no content available for the artifact. E.g. a logical package configuration without physical files.
+     *         This may require provision of download urls or additional content (however at a later stage)
+     *     </li>
+     *     <li>
+     *         Artifacts that have been unpacked for scanning the content (classification contains 'scan')
+     *     </li>
+     * </ul>
+     *
+     * @param inventory The inventory to check for KEY_ARCHIVE_PATH completeness.
+     */
+    private void checkCompletenessOfArchivePath(Inventory inventory) {
+        for (final Artifact artifact : inventory.getArtifacts()) {
+            final String archivePath = artifact.get(Constants.KEY_ARCHIVE_PATH);
+            final String contentChecksum = artifact.get(KEY_CONTENT_CHECKSUM);
+            final String checksum = artifact.getChecksum();
+
+            final boolean hasChecksum = StringUtils.isNotBlank(checksum);
+            final boolean hasContentChecksum = StringUtils.isNotBlank(contentChecksum);
+
+            // deep scanned artifacts are not further scanned. It would be good to get an aggregated view however
+            if (ArtifactUtils.hasScanClassification(artifact)) {
+                if (hasContentChecksum || hasChecksum) {
+                    LOG.warn("Artifact {} with scan classification must have checksum and a content checksum.", artifact);
+                }
+                continue;
+            }
+
+            // skipped artifacts have no archive path (no redundant aggregation)
+            if (hasSkipAggregationDirective(artifact)) continue;
+
+            if (StringUtils.isBlank(archivePath)) {
+                // only report issue, when we have a checksum; implicitly excluded shaded subcomponents from being reported
+                if (hasContentChecksum || hasChecksum) {
+                    LOG.warn("Artifact {} with file content does not have an archive path! " +
+                            "Validate that the component patterns for this process are complete.", artifact);
+                }
+            }
+        }
+    }
+
 }

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/filescan/tasks/ArtifactUnwrapTask.java

@@ -202,6 +202,9 @@ private void postProcessUnwrapped(Artifact artifact, File file, File targetFolde
         try {
             deriveType(artifact, file);
 
+            // unwrapped items mit aggregate directive skip
+            artifact.set(Constants.KEY_AGGREGATE_DIRECTIVE, AGGREGATE_DIRECTIVE_SKIP);
+
             postProcessUnwrappedSavedContainer(targetFolder);
         } catch (Exception e) {
             issues.add("Detected saved container, but unable to postprocess.");

libraries/ae-inventory-processor/src/main/java/org/metaeffekt/core/inventory/processor/model/Constants.java

@@ -114,6 +114,9 @@ public final class Constants {
     public static final String ARTIFACT_TYPE_ARCHIVE = "archive";
     public static final String ARTIFACT_TYPE_COMPOSITE = "composite";
 
+    public static final String KEY_AGGREGATE_DIRECTIVE = "Aggregate Directive";
+    public static final String AGGREGATE_DIRECTIVE_SKIP = "skip";
+
     @Deprecated
     public static final String ARTIFACT_TYPE_NODEJS_MODULE = "nodejs-module";
 
@@ -130,6 +133,7 @@ public final class Constants {
     public static final String DOT_BOWER_JSON = DOT + BOWER_JSON;
     public static final String DOT_PACKAGE_LOCK_JSON = DOT + PACKAGE_LOCK_JSON;
 
+
     private Constants() {
     }
 }

libraries/ae-test-common/src/main/java/org/metaeffekt/core/itest/common/setup/AbstractTestSetup.java

@@ -35,24 +35,24 @@ public abstract class AbstractTestSetup implements TestSetup {
 
     String name;
     private Inventory inventory;
-    private String myDir = "";
+    private String localDir = "";
 
     private String referenceInventory = "";
 
     private File aggregationDir;
 
     public String getDownloadFolder() {
-        return TestConfig.getDownloadFolder() + myDir;
+        return TestConfig.getDownloadFolder() + localDir;
     }
 
     @Override
     public String getScanFolder() {
-        return TestConfig.getScanFolder() + myDir;
+        return TestConfig.getScanFolder() + localDir;
     }
 
     @Override
     public String getInventoryFolder() {
-        return TestConfig.getInventoryFolder() + myDir;
+        return TestConfig.getInventoryFolder() + localDir;
     }
 
     public TestSetup setSource(String url) {
@@ -63,7 +63,7 @@ public TestSetup setSource(String url) {
     @Override
     public TestSetup setName(String testName) {
         this.name = testName.replace("org.metaeffekt.core.itest.", "");
-        this.myDir = name.replace(".", "/") + "/";
+        this.localDir = name.replace(".", "/") + "/";
         return this;
     }
 
@@ -72,6 +72,7 @@ public boolean clear() throws Exception {
         FileUtils.deleteDirectory(new File(getInventoryFolder()));
         FileUtils.deleteDirectory(new File(getDownloadFolder()));
         FileUtils.deleteDirectory(new File(getScanFolder()));
+        FileUtils.deleteDirectory(new File(getAggregationDir()));
         return true;
     }
 
@@ -129,8 +130,8 @@ public String getSha256Hash() {
     }
 
     @Override
-    public File getAggregationDir() {
-        return aggregationDir;
+    public String getAggregationDir() {
+        return TestConfig.getAggregationFolder() + localDir;
     }
 
     // NOTE: this is used by artifact-analysis, please don't remove

libraries/ae-test-common/src/main/java/org/metaeffekt/core/itest/common/setup/TestConfig.java

@@ -20,11 +20,16 @@ public class TestConfig {
     private static final String DOWNLOAD_FOLDER = ".test/downloads/";
     private static final String SCAN_FOLDER = "target/.test/scan/";
     private static final String INVENTORY_FOLDER = "target/.test/inventory/";
+    private static final String AGGREGATION_FOLDER = "target/.test/aggregation/";
 
     public static String getDownloadFolder() {
         return DOWNLOAD_FOLDER;
     }
 
+    public static String getAggregationFolder() {
+        return AGGREGATION_FOLDER;
+    }
+
     public static String getScanFolder() {
         return SCAN_FOLDER;
     }

libraries/ae-test-common/src/main/java/org/metaeffekt/core/itest/common/setup/TestSetup.java

@@ -50,5 +50,5 @@ public interface TestSetup {
 
     void setAggregationDir(File aggregationDir);
 
-    File getAggregationDir();
+    String getAggregationDir();
 }

libraries/ae-test-common/src/main/java/org/metaeffekt/core/itest/common/setup/UrlBasedTestSetup.java

@@ -42,7 +42,7 @@ public boolean inventorize(boolean overwrite) throws Exception {
         new File(getScanFolder()).mkdirs();
         final File scanInputDir = new File(getDownloadFolder());
         final File scanDir = new File(getScanFolder());
-        final File aggregationDir = getAggregationDir();
+        final File aggregationDir = new File(getAggregationDir());
 
         String[] scanIncludes = new String[]{"**/*"};
         String[] scanExcludes = new String[]{

tests/ae-inventory-container-itest/src/test/java/org/metaeffekt/core/itest/container/ContainerDumpSetup.java

@@ -24,6 +24,8 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
+import java.util.stream.Collectors;
 
 import static org.metaeffekt.core.itest.common.setup.TestConfig.getDownloadFolder;
 import static org.metaeffekt.core.itest.common.setup.TestConfig.getScanFolder;
@@ -113,10 +115,10 @@ private static String executeCommand(String[] commands) throws IOException, Inte
             }
         }
 
-        int exitVal = process.waitFor(); // Wait for the process to complete.
-        if (exitVal != 0) {
+        final int exitValue = ExecUtils.waitForProcessToTerminate(process, Arrays.stream(commands).collect(Collectors.joining(" ")));
+        if (exitValue != 0) {
             // Handle the case where the process did not complete successfully.
-            throw new IOException("Command execution failed with exit code " + exitVal);
+            throw new IOException("Command execution failed with exit code " + exitValue);
         }
 
         return output.toString().trim(); // Return the accumulated output, which includes the container ID at the end.