diff --git a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java index 29205d6b..0b4afff1 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java @@ -104,7 +104,9 @@ public enum LogMessages { "When reading yarn license info from file '{}' there was at least one virtual package encountered. Check if package resolution is correct"), // MODERN_YARN_PATCHED_PACKAGE(70, "When reading yarn license info from file '{}' there was at least one patched package encountered. Processing only the base package, not the patched version."), // - FAILED_READING_FILE(71, "Reading file '{}' failed"); + FAILED_READING_FILE(71, "Reading file '{}' failed"), // + EMPTY_PACKAGE_URL(72, "The package URL is null or empty."), // + EMPTY_PACKAGE_PATH(73, "The package path is null or empty."); private final String message; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoInventoryProcessor.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoInventoryProcessor.java index 1def280f..7edd82f4 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoInventoryProcessor.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoInventoryProcessor.java @@ -12,6 +12,7 @@ import com.devonfw.tools.solicitor.InventoryProcessor; import com.devonfw.tools.solicitor.common.LogMessages; import com.devonfw.tools.solicitor.common.SolicitorRuntimeException; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; import com.devonfw.tools.solicitor.model.ModelFactory; import com.devonfw.tools.solicitor.model.ModelRoot; import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent; @@ -145,6 +146,8 @@ private Statistics processApplicationComponent(ApplicationComponent ac) { } } catch (ComponentInfoAdapterException e) { throw new SolicitorRuntimeException("Exception when reading component info data source", e); + } catch (CurationInvalidException e) { + throw new SolicitorRuntimeException("Curation data invalid when reading component info data source", e); } if (componentInfo == null) { // all adapters disabled diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoProvider.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoProvider.java index 6ddc9d7e..eebc5c82 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoProvider.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/ComponentInfoProvider.java @@ -1,5 +1,7 @@ package com.devonfw.tools.solicitor.componentinfo; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; + /** * Provides {@link ComponentInfo} for components given by their PackageURL. Subinterfaces further specify if the * {@link ComponentInfo} is already curated or not. @@ -19,8 +21,9 @@ public interface ComponentInfoProvider { * {@link ComponentInfo#getComponentInfoData()} on the returned object will return null. * @throws ComponentInfoAdapterException if there was an exception when reading the data. In case that there is no * data available no exception will be thrown. Instead null will be returned in such a case. + * @throws CurationInvalidException if the curation data is not valid */ ComponentInfo getComponentInfo(String packageUrl, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException; + throws ComponentInfoAdapterException, CurationInvalidException; } \ No newline at end of file diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractCurationProviderBase.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractCurationProviderBase.java new file mode 100644 index 00000000..cf4912dd --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractCurationProviderBase.java @@ -0,0 +1,56 @@ +package com.devonfw.tools.solicitor.componentinfo.curation; + +import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; +import com.devonfw.tools.solicitor.componentinfo.CurationDataHandle; +import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; + +/** + * Abstract base implementation of {@link CurationProvider} to be used as starting point for concrete implementations. + * + */ +public abstract class AbstractCurationProviderBase implements CurationProvider { + + /** + * The constructor. + */ + public AbstractCurationProviderBase() { + + super(); + } + + /** + * Fetches curation data for a given package URL and returns it as a ComponentInfoCuration object. + *

+ * Method to be implemented in subclasses. + * + * It has the same functionality as {@link CurationProvider#findCurations(String, CurationDataHandle)} but does not + * require validating the fetched curation data. + * + * @param packageUrl The package URL for which curation data is to be fetched. + * @param curationDataHandle identifies which source should be used for the curation data. + * @return A ComponentInfoCuration object containing curation data. + * @throws ComponentInfoAdapterException If any errors occur during the retrieval or parsing of curation data. + * @throws ComponentInfoAdapterNonExistingCurationDataSelectorException If the specified curation data selector does + * not exist. + */ + protected abstract ComponentInfoCuration doFindCurations(String packageUrl, CurationDataHandle curationDataHandle) + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException; + + /** + * Implementation of {@link CurationProvider#findCurations(String, CurationDataHandle)} which delegates the fetching + * of the curations to (abstract) method {@link #doFindCurations(String, CurationDataHandle)} and then validates the + * result before returning. + */ + @Override + public ComponentInfoCuration findCurations(String packageUrl, CurationDataHandle curationDataHandle) + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException, + CurationInvalidException { + + ComponentInfoCuration curation = doFindCurations(packageUrl, curationDataHandle); + if (curation != null) { + curation.validate(); + } + return curation; + } + +} \ No newline at end of file diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractHierarchicalCurationProvider.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractHierarchicalCurationProvider.java new file mode 100644 index 00000000..539683f1 --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractHierarchicalCurationProvider.java @@ -0,0 +1,241 @@ +package com.devonfw.tools.solicitor.componentinfo.curation; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.devonfw.tools.solicitor.common.LogMessages; +import com.devonfw.tools.solicitor.common.packageurl.AllKindsPackageURLHandler; +import com.devonfw.tools.solicitor.common.packageurl.PackageURLHandler; +import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; +import com.devonfw.tools.solicitor.componentinfo.CurationDataHandle; +import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; +import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; + +/** + * Abstract base implementation of {@link CurationProvider} which supports storing curations in a hierarchical structure + * so that curations are not required to be defined for specific component versions but might be defined for groups of + * components. The hierarchy is derived from the PackageURL, see {@link PackageURLHandler#pathFor(String)}. The + * {@link CurationProvider#findCurations(String, CurationDataHandle)} method of this class requires the + * {@link CurationDataHandle} to be a {@link SelectorCurationDataHandle}. + * + */ +public abstract class AbstractHierarchicalCurationProvider extends AbstractCurationProviderBase { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractHierarchicalCurationProvider.class); + + private final AllKindsPackageURLHandler packageURLHandler; + + /** + * The constructor. + * + * @param packageURLHandler the required packageURLHandler + */ + public AbstractHierarchicalCurationProvider(AllKindsPackageURLHandler packageURLHandler) { + + super(); + this.packageURLHandler = packageURLHandler; + } + + /** + * Fetches curation data for a given package URL from a curation repository and returns it as a ComponentInfoCuration + * object. + * + *

+ * This method fetches curation data for a given package URL from a Repository and returns it as a + * ComponentInfoCuration object. Fetching from a hierarchy and merging the different curation fragments is supported. + * In case that data is requested for a non default curationDataSelector which does not exist will result in an + * exception to be thrown. + *

+ * + * @param packageUrl The package URL for which curation data is to be fetched. + * @param curationDataHandle This has to be a {@link SelectorCurationDataHandle} and specifies the (alternative) + * location to take the curation data from. If the curationDataSelector is set to "none" then no curation will + * be returned. + * @return A ComponentInfoCuration object containing curation data (unvalidated). null if no curation + * data exists or the curationDataSelector of the curationDataHandle is set to "none". + * @throws ComponentInfoAdapterException If any errors occur during the retrieval of curation data. + * @throws ComponentInfoAdapterNonExistingCurationDataSelectorException If the specified curationDataSelector does not + * exist. + */ + @Override + protected ComponentInfoCuration doFindCurations(String packageUrl, CurationDataHandle curationDataHandle) + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException { + + LOG.debug("Received packageUrl: '{}'", packageUrl); + if (packageUrl == null || packageUrl.isEmpty()) { + LOG.error(LogMessages.EMPTY_PACKAGE_URL.msg()); + throw new ComponentInfoAdapterException("The package URL cannot be null or empty."); + } + + String curationDataSelector = ((SelectorCurationDataHandle) curationDataHandle).getCurationDataSelector(); + // Check if curation should not be applied + if ("none".equalsIgnoreCase(curationDataSelector)) { + LOG.debug("Retrieving curation is disabled for packageUrl '{}' due to curationDataSelector having value 'none'", + packageUrl); + return null; + } + String effectiveCurationDataSelector; + effectiveCurationDataSelector = determineEffectiveCurationDataSelector(curationDataSelector); + validateEffectiveCurationDataSelector(effectiveCurationDataSelector); + + // Parse the package URL to get the path to the curation file in the curation repository + String pathFromPackageUrl = this.packageURLHandler.pathFor(packageUrl); + LOG.debug("Parsed package path: '{}'", pathFromPackageUrl); + if (pathFromPackageUrl == null || pathFromPackageUrl.isEmpty()) { + LOG.error(LogMessages.EMPTY_PACKAGE_PATH.msg()); + throw new ComponentInfoAdapterException( + "The package path parsed from the package URL is null or empty for the package URL: " + packageUrl); + } + + List data = null; + try { + data = fetchAllDataFromCurationRepository(effectiveCurationDataSelector, pathFromPackageUrl); + } catch (CurationInvalidException e) { + throw new ComponentInfoAdapterException("Error while mapping JSON content", e.getCause()); + } + + if (data.isEmpty()) { + assureCurationDataSelectorAvailable(effectiveCurationDataSelector); + LOG.debug("NO curations found in Curations Repository for '{}' with curationDataSelector '{}'", packageUrl, + effectiveCurationDataSelector); + return null; + } + LOG.debug("Curations found in Curations Repository for '{}' with curationDataSelector '{}'", packageUrl, + effectiveCurationDataSelector); + + ComponentInfoCuration merged = mergeCurationData(data); + return merged; + } + + /** + * Fetches a single curation object (or curation fragment) from the hierarchy for the defined component. + *

+ * To be implemented by concrete subclasses. + * + * @param effectiveCurationDataSelector the effective curationDataSelector + * @param pathFragmentWithinRepo the path of the curation data or fragment. This will be either the value returned by + * {@link PackageURLHandler#pathFor(String)} or a (trailing) subpath. + * @return The found curation object (might be a fragment). null if nothing is defined. + * @throws ComponentInfoAdapterException if there was any unforeseen error when trying to retrieve the curation + * object. + * @throws CurationInvalidException if the curation data was malformed + */ + protected abstract ComponentInfoCuration fetchCurationFromRepository(String effectiveCurationDataSelector, + String pathFragmentWithinRepo) throws ComponentInfoAdapterException, CurationInvalidException; + + /** + * Validate the effective curationDataSelector. This needs to check is syntactically correct and might not trigger any + * unwanted effects e.g. like path traversal. + *

+ * To be implemented by concrete subclasses. + * + * @param effectiveCurationDataSelector the curationDataSelector to be validated + * @throws ComponentInfoAdapterNonExistingCurationDataSelectorException if validation fails + */ + protected abstract void validateEffectiveCurationDataSelector(String effectiveCurationDataSelector) + throws ComponentInfoAdapterNonExistingCurationDataSelectorException; + + /** + * Determine the effective curationDataSelector. This specifically needs to handle the case that the value + * null is given which needs to be interpreted as "take the default". + *

+ * To be implemented by concrete subclasses. + * + * @param curationDataSelector the incoming value. Might be null. + * @return the effective curationDataSelector. Must not be null. + */ + protected abstract String determineEffectiveCurationDataSelector(String curationDataSelector); + + /** + * Method which checks if the given effecticeCurationDataSelector actually exists (is defined). The method will be + * called in case that no curations could be found to make sure that this was not due to a wrong value given for the + * effectiveCurationDataSelector. + *

+ * To be implemented by concrete subclasses. + * + * @param effectiveCurationDataSelector the value to check for existence + * @throws ComponentInfoAdapterException in case that there was any unforeseen exception when trying to check the + * existence of the curationDataSelector. + * @throws ComponentInfoAdapterNonExistingCurationDataSelectorException in case that the given + * effectiveCurationDataSelector does not exist + */ + protected abstract void assureCurationDataSelectorAvailable(String effectiveCurationDataSelector) + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException; + + /** + * Determine if curations should be fetched in a hierarchical manner or only for the concrete version of a component. + * The default implementation returns true. Subclasses might override this if they do not support + * hierarchies or want to make this configurable. + * + * @return true if the hierarchy should be evaluated, false otherwise, + */ + protected boolean isHierarchyEvaluation() { + + return true; + } + + /** + * Merges the list of found ComponentInfoCuration objects into a resulting curation. + * + * @param data the list of curation (framents) + * @return the resulting curation object + */ + private ComponentInfoCuration mergeCurationData(List data) { + + ComponentInfoCuration mergedCuration = null; + for (ComponentInfoCuration oneCurationData : data) { + mergedCuration = CurationUtil.merge(mergedCuration, oneCurationData); + } + + return mergedCuration; + } + + /** + * Iterates though the hierarchy (optionally) and fetches the curation data objects/fragments via + * {@link #fetchCurationFromRepository(String, String)}. Returns them as a list with the most specific ones at the + * end. + * + * @param effectiveCurationDataSelector the effective curationDataSelector + * @param pathFromPackageUrl the path as determined from the PackageURL (see + * {@link PackageURLHandler#pathFor(String)}) + * @return the list of found curation objects/fragments. Sorting of list is with increasing priority (curations being + * less specific appear first). + * @throws ComponentInfoAdapterException if there was some unforseen issue when accessing the coration repository + * @throws CurationInvalidException if some curation object was malformed + */ + private List fetchAllDataFromCurationRepository(String effectiveCurationDataSelector, + String pathFromPackageUrl) throws ComponentInfoAdapterException, CurationInvalidException { + + List curationsData = new ArrayList<>(); + + String[] pathElements; + if (isHierarchyEvaluation()) { + pathElements = pathFromPackageUrl.split("/"); + } else { + // only take the complete path - do not go through hierarchy + pathElements = new String[] { pathFromPackageUrl }; + } + + String effectivePath = null; + + for (String element : pathElements) { + if (effectivePath == null) { + effectivePath = element; + } else { + effectivePath = effectivePath + "/" + element; + } + ComponentInfoCuration data = fetchCurationFromRepository(effectiveCurationDataSelector, effectivePath); + if (data != null) { + curationsData.add(data); + LOG.debug("Curations found on path '{}' within curation repo.", effectivePath); + } else { + LOG.debug("No Curations found on path '{}' within curation repo.", effectivePath); + } + } + return curationsData; + } + +} diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCurator.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCurator.java index 7a964784..41f5bf29 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCurator.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCurator.java @@ -17,8 +17,9 @@ public interface ComponentInfoCurator { * @param curationDataHandle identifies which source should be used for the curation data. * @return the curated component info or - if no curation are done - the incoming object * @throws ComponentInfoAdapterException if the curation could not be read + * @throws CurationInvalidException if the curation data is not valid */ ComponentInfo curate(ComponentInfo componentInfo, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException; + throws ComponentInfoAdapterException, CurationInvalidException; } \ No newline at end of file diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCuratorImpl.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCuratorImpl.java index a8f47574..59474c68 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCuratorImpl.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/ComponentInfoCuratorImpl.java @@ -52,17 +52,17 @@ public ComponentInfoCuratorImpl(CurationProvider curationProvider, * @param curationDataHandle identifies which source should be used for the curation data. * @return the curated component info * @throws ComponentInfoAdapterException if the curation could not be read + * @throws CurationInvalidException if the curation data is not valid */ @Override public ComponentInfo curate(ComponentInfo componentInfo, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException { + throws ComponentInfoAdapterException, CurationInvalidException { ComponentInfoCuration foundCuration = this.curationProvider.findCurations(componentInfo.getPackageUrl(), curationDataHandle); if (foundCuration != null) { DefaultComponentInfoImpl componentInfoImpl = new DefaultComponentInfoImpl(componentInfo); applyFoundCurations(componentInfoImpl, foundCuration); - componentInfoImpl.setDataStatus(DataStatusValue.CURATED); return componentInfoImpl; } else { return componentInfo; @@ -76,9 +76,11 @@ private void applyFoundCurations(DefaultComponentInfoImpl componentInfo, Compone for (String copyright : curation.getCopyrights()) { componentInfo.getComponentInfoData().addCopyright(copyright); } + componentInfo.setDataStatus(DataStatusValue.CURATED); } if (curation.getUrl() != null) { componentInfo.getComponentInfoData().setSourceRepoUrl(curation.getUrl()); + componentInfo.setDataStatus(DataStatusValue.CURATED); } if (curation.getLicenses() != null) { componentInfo.getComponentInfoData().clearLicenses(); @@ -95,6 +97,7 @@ private void applyFoundCurations(DefaultComponentInfoImpl componentInfo, Compone licenseInfo.setGivenLicenseText(givenLicenseText); componentInfo.getComponentInfoData().addLicense(licenseInfo); } + componentInfo.setDataStatus(DataStatusValue.CURATED); } } diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CuratingComponentInfoAdapter.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CuratingComponentInfoAdapter.java index 30b84db5..1d07779f 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CuratingComponentInfoAdapter.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CuratingComponentInfoAdapter.java @@ -55,10 +55,11 @@ public CuratingComponentInfoAdapter(FilteredComponentInfoProvider filteredCompon * @return the data derived from the scancode results after applying any defined curation. * @throws ComponentInfoAdapterException if there was an exception when reading the data. In case that there is no * data available no exception will be thrown. Instead null will be return in such a case. + * @throws CurationInvalidException if the curation data is not valid */ @Override public ComponentInfo getComponentInfo(String packageUrl, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException { + throws ComponentInfoAdapterException, CurationInvalidException { if (isFeatureActive()) { diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationInvalidException.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationInvalidException.java new file mode 100644 index 00000000..c1093414 --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationInvalidException.java @@ -0,0 +1,28 @@ +package com.devonfw.tools.solicitor.componentinfo.curation; + +/** + * Exception which indicates that curation data is invalid. + */ +public class CurationInvalidException extends Exception { + + /** + * The constructor. + * + * @param message the message + */ + public CurationInvalidException(String message) { + + super(message); + } + + /** + * The constructor. + * + * @param cause the underlying cause + */ + public CurationInvalidException(Throwable cause) { + + super("Curation data is invalid", cause); + } + +} \ No newline at end of file diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationProvider.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationProvider.java index 29880a93..ff3dabaf 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationProvider.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationProvider.java @@ -20,8 +20,10 @@ public interface CurationProvider { * @throws ComponentInfoAdapterException if something unexpected happens * @throws ComponentInfoAdapterNonExistingCurationDataSelectorException if the specified curationDataSelector could * not be resolved + * @throws CurationInvalidException if the curation data is not valid */ ComponentInfoCuration findCurations(String packageUrl, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException; + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException, + CurationInvalidException; } \ No newline at end of file diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationUtil.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationUtil.java new file mode 100644 index 00000000..84df479a --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationUtil.java @@ -0,0 +1,110 @@ +package com.devonfw.tools.solicitor.componentinfo.curation; + +import java.util.ArrayList; +import java.util.List; + +import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * Helper methods for working with curation data. + */ +public class CurationUtil { + + private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + + /** + * Parses curation data from a YAML string into a ComponentInfoCuration object. + * + * @param curationYaml The YAML data to be parsed. + * @return A ComponentInfoCuration object containing parsed curation data. + * @throws CurationInvalidException If any errors occur during YAML parsing. + */ + public static ComponentInfoCuration curationDataFromString(String curationYaml) throws CurationInvalidException { + + ComponentInfoCuration result; + try { + // Use Jackson YAML mapper to parse YAML into an object + result = yamlMapper.readValue(curationYaml, ComponentInfoCuration.class); + } catch (JsonProcessingException e) { + throw new CurationInvalidException(e); + } + if (result.getName() == null || result.getName().isEmpty()) { + throw new CurationInvalidException("The name attribute of the curation data must not be null or empty"); + } + return result; + } + + /** + * Private constructor prohibits instantiation + */ + private CurationUtil() { + + } + + /** + * Merges two ComponentInfoCuration objects. The "second" argument represents the more specific one (with higher + * priority): name and url of the second one supersede the first one. If data is concatenated as list, then the + * elements from the second are are front of the list, the elements from the first are at the back. + * + * @param first the first object (less spoecific / lower priority) + * @param second the second object (more specific / higher priority) + * @return the merged result + */ + @SuppressWarnings("unchecked") + public static ComponentInfoCuration merge(ComponentInfoCuration first, ComponentInfoCuration second) { + + if (first == null) { + return second; + } + if (second == null) { + return first; + } + ComponentInfoCuration merged = new ComponentInfoCuration(); + merged.setName(second.getName() != null ? second.getName() : first.getName()); // latest wins + merged.setNote(join(" / ", second.getNote(), first.getNote())); + merged.setUrl(second.getUrl() != null ? second.getUrl() : first.getUrl()); // latest wins + merged.setCopyrights(join(second.getCopyrights(), first.getCopyrights())); + merged.setLicenses(join(second.getLicenses(), first.getLicenses())); + merged.setExcludedPaths(join(second.getExcludedPaths(), first.getExcludedPaths())); + merged.setLicenseCurations(join(second.getLicenseCurations(), first.getLicenseCurations())); + merged.setCopyrightCurations(join(second.getCopyrightCurations(), first.getCopyrightCurations())); + return merged; + } + + private static String join(String delimiter, String arg1, String arg2) { + + if (arg1 == null && arg2 == null) { + return null; + } + if (arg1 != null) { + StringBuffer sb = new StringBuffer(arg1); + if (arg2 != null) { + sb.append(delimiter).append(arg2); + } + return sb.toString(); + } else { + return arg2; + } + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static List join(List arg1, List arg2) { + + if (arg1 == null && arg2 == null) { + return null; + } + if (arg1 != null) { + List result = new ArrayList(arg1); + if (arg2 != null) { + result.addAll(arg2); + } + return result; + } else { + return arg2; + } + } + +} diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProvider.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProvider.java index b72fd9ab..a9f38d29 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProvider.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProvider.java @@ -14,8 +14,6 @@ import com.devonfw.tools.solicitor.common.LogMessages; import com.devonfw.tools.solicitor.common.packageurl.AllKindsPackageURLHandler; import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; -import com.devonfw.tools.solicitor.componentinfo.CurationDataHandle; -import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; import com.devonfw.tools.solicitor.componentinfo.curation.model.CurationList; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,7 +25,7 @@ * */ @Component -public class SingleFileCurationProvider implements CurationProvider { +public class SingleFileCurationProvider extends AbstractHierarchicalCurationProvider { private static final Logger LOG = LoggerFactory.getLogger(SingleFileCurationProvider.class); private static final ObjectMapper yamlMapper; @@ -41,8 +39,6 @@ public class SingleFileCurationProvider implements CurationProvider { private boolean curationsExistenceLogged; - private AllKindsPackageURLHandler packageURLHandler; - /** * The constructor. * @@ -51,31 +47,25 @@ public class SingleFileCurationProvider implements CurationProvider { @Autowired public SingleFileCurationProvider(AllKindsPackageURLHandler packageURLHandler) { - this.packageURLHandler = packageURLHandler; + super(packageURLHandler); } /** - * Return the curation data for a given package. + * Sets curationsFileName. * - * @param packageUrl identifies the package - * @param curationDataHandle identifies which source should be used for the curation data. - * @return the curation data if it exists. null if no curation exist for the package or the - * curationDataSelector was given as "none". - * @throws ComponentInfoAdapterException if something unexpected happens + * @param curationsFileName new value of curationsFileName. */ + @Value("${solicitor.scancode.curations-filename}") + public void setCurationsFileName(String curationsFileName) { + + this.curationsFileName = curationsFileName; + } + @Override - public ComponentInfoCuration findCurations(String packageUrl, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException { - - // SelectorCurationDataHandle is the only implementation supported here. - String curationDataSelector = ((SelectorCurationDataHandle) curationDataHandle).getCurationDataSelector(); - // Return null if curationDataSelector is "none" - if ("none".equalsIgnoreCase(curationDataSelector)) { - return null; - } - ComponentInfoCuration foundCuration = null; + protected ComponentInfoCuration fetchCurationFromRepository(String effectiveCurationDataSelector, + String pathFragmentWithinRepo) throws ComponentInfoAdapterException, CurationInvalidException { - String packagePathPart = this.packageURLHandler.pathFor(packageUrl); + ComponentInfoCuration foundCuration = null; File curationsFile = new File(this.curationsFileName); if (!curationsFile.exists()) { @@ -96,7 +86,7 @@ public ComponentInfoCuration findCurations(String packageUrl, CurationDataHandle for (ComponentInfoCuration curation : curationList.getArtifacts()) { String component = curation.getName(); - if (component.equals(packagePathPart)) { + if (component.equals(pathFragmentWithinRepo)) { foundCuration = curation; break; } @@ -109,15 +99,26 @@ public ComponentInfoCuration findCurations(String packageUrl, CurationDataHandle return foundCuration; } - /** - * Sets curationsFileName. - * - * @param curationsFileName new value of curationsFileName. - */ - @Value("${solicitor.scancode.curations-filename}") - public void setCurationsFileName(String curationsFileName) { + @Override + protected void validateEffectiveCurationDataSelector(String effectiveCurationDataSelector) + throws ComponentInfoAdapterNonExistingCurationDataSelectorException { + + // as the curationDataSelector is not supported/used there is nothing to do here + } + + @Override + protected String determineEffectiveCurationDataSelector(String curationDataSelector) { + + // actually this value is unused in the class + return "-"; + } + + @Override + protected void assureCurationDataSelectorAvailable(String effectiveCurationDataSelector) + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException { + + // as the curationDataSelector is not supported/used there is nothing to do here - this.curationsFileName = curationsFileName; } } diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/ComponentInfoCuration.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/ComponentInfoCuration.java index 94e85453..7ccd7a9e 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/ComponentInfoCuration.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/ComponentInfoCuration.java @@ -2,6 +2,7 @@ import java.util.List; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; import com.fasterxml.jackson.annotation.JsonInclude; /** @@ -23,6 +24,10 @@ public class ComponentInfoCuration { private List excludedPaths; + private List licenseCurations; + + private List copyrightCurations; + /** * The constructor. */ @@ -126,4 +131,56 @@ public void setExcludedPaths(List excludedPaths) { this.excludedPaths = excludedPaths; } + /** + * @return licenseCurations + */ + public List getLicenseCurations() { + + return this.licenseCurations; + } + + /** + * @param licenseCurations new value of {@link #getLicenseCurations}. + */ + public void setLicenseCurations(List licenseCurations) { + + this.licenseCurations = licenseCurations; + } + + /** + * @return copyrightCurations + */ + public List getCopyrightCurations() { + + return this.copyrightCurations; + } + + /** + * @param copyrightCurations new value of {@link #getCopyrightCurations}. + */ + public void setCopyrightCurations(List copyrightCurations) { + + this.copyrightCurations = copyrightCurations; + } + + /** + * Validates the curation data. + * + * @throws CurationInvalidException indicates that the curation data is not valid. + */ + public void validate() throws CurationInvalidException { + + if (this.licenseCurations != null) { + for (LicenseCuration lc : this.licenseCurations) { + lc.validate(); + } + } + if (this.copyrightCurations != null) { + for (CopyrightCuration cc : this.copyrightCurations) { + cc.validate(); + } + } + + } + } diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CopyrightCuration.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CopyrightCuration.java new file mode 100644 index 00000000..b4d2e9ee --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CopyrightCuration.java @@ -0,0 +1,214 @@ +package com.devonfw.tools.solicitor.componentinfo.curation.model; + +import java.util.regex.Pattern; + +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Copyright curation. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class CopyrightCuration { + + private CurationOperation operation; + + private Pattern path; + + private Pattern oldCopyright; + + private String newCopyright; + + private String comment; + + /** + * The constructor. + */ + public CopyrightCuration() { + + } + + /** + * @return operation + */ + public CurationOperation getOperation() { + + return this.operation; + } + + /** + * Returns the regular expressing which specifies the file path(s) to which this curation should apply. + * + * @return path + */ + public String getPath() { + + return this.path.toString(); + } + + /** + * Returns the regular expressing which specifies the copyright(s) to which this curation should apply. + * + * @return copyright + */ + public String getOldCopyright() { + + return this.oldCopyright.toString(); + } + + /** + * @return newCopyright + */ + public String getNewCopyright() { + + return this.newCopyright; + } + + /** + * Returns the + * + * @return comment + */ + public String getComment() { + + return this.comment; + } + + /** + * @param operation new value of {@link #getOperation}. + */ + public void setOperation(CurationOperation operation) { + + this.operation = operation; + } + + /** + * Sets the regular expression for specifying the file paths(s) to which this curation should apply. + * + * @param path new value of {@link #getPath}. + */ + public void setPath(String path) { + + this.path = Pattern.compile(path); + } + + /** + * Sets the regular expression for specifying the copyright(s) to which this curation should apply. + * + * @param oldCopyright new value of {@link #getOldCopyright}. + */ + public void setOldCopyright(String oldCopyright) { + + this.oldCopyright = Pattern.compile(oldCopyright, Pattern.DOTALL); + } + + /** + * @param newCopyright new value of {@link #getNewCopyright}. + */ + public void setNewCopyright(String newCopyright) { + + this.newCopyright = newCopyright; + } + + /** + * Sets the comment describing the curation + * + * @param comment new value of {@link #getComment}. + */ + public void setComment(String comment) { + + this.comment = comment; + } + + /** + * Check if this curation matches the given data. If any of the conditions are not set (via the appropriate setters or + * by reading the data from yaml) then the condition will be ignored. + * + * @param path the path to check + * @param copyright the ruleIdentifier to check + * @return true if the conditions match, false otherwise. + */ + @SuppressWarnings("hiding") + public boolean matches(String path, String copyright) { + + if (this.operation != CurationOperation.REMOVE && this.operation != CurationOperation.REPLACE) { + // this method should only match in case of REMOVE or REPLACE + return false; + } + if (this.path != null && (path == null || !this.path.matcher(path).matches())) { + return false; + } + if (this.oldCopyright != null && (copyright == null || !this.oldCopyright.matcher(copyright).matches())) { + return false; + } + return true; + + } + + /** + * Check if this a ADD curation and matches the given path. If any of the conditions are not set (via the appropriate + * setters or by reading the data from yaml) then the condition will be ignored. + * + * @param path the path to check + * @return true if this is an ADD curation and the conditions match, false otherwise. + */ + @SuppressWarnings("hiding") + public boolean matches(String path) { + + if (this.operation != CurationOperation.ADD) { + // no match if it is not an ADD curation + return false; + } + + if (this.path == null) { + // if path condition is not set then this will only match if being called with null argument. + return path == null; + } else { + if (path == null) { + return false; + } else { + // if path condition is set the check if the given path matches the Pattern + return this.path.matcher(path).matches(); + } + } + + } + + /** + * Validates if the data of this object is consistent. + * + * @throws CurationInvalidException if the object is invalid + */ + public void validate() throws CurationInvalidException { + + if (this.operation == null) { + throw new CurationInvalidException("Operation must not be null for copyright curation"); + } + if (this.operation == CurationOperation.REMOVE || this.operation == CurationOperation.REPLACE) { + if (this.path == null && this.oldCopyright == null) { + throw new CurationInvalidException( + "For REMOVE/REPLACE copyright curation at least one condition must be defined"); + } + } + if (this.operation == CurationOperation.REMOVE) { + if (this.newCopyright != null) { + throw new CurationInvalidException("For REMOVE copyright curation the newCopyright must not be set"); + } + } + if (this.operation == CurationOperation.REPLACE) { + if (this.newCopyright == null) { + throw new CurationInvalidException("For REPLACE copyright curation newCopyright must be set"); + } + } + if (this.operation == CurationOperation.ADD) { + if (this.oldCopyright != null) { + throw new CurationInvalidException("For ADD copyright curation oldCopyright must not be defined"); + } + if (this.newCopyright == null) { + throw new CurationInvalidException("For ADD copyright curation newCopyright must be set"); + } + } + + } + +} diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CurationOperation.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CurationOperation.java new file mode 100644 index 00000000..e55aaa02 --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CurationOperation.java @@ -0,0 +1,20 @@ +package com.devonfw.tools.solicitor.componentinfo.curation.model; + +/** + * The curation operation to be performed. + */ +public enum CurationOperation { + /** + * Remove / Ignore a license or copyright finding. + */ + REMOVE, + /** + * Replace a license or copyright finding with a different one. + */ + REPLACE, + /** + * Add a license or copyright. + */ + ADD + +} diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/LicenseCuration.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/LicenseCuration.java new file mode 100644 index 00000000..8fec7eec --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/curation/model/LicenseCuration.java @@ -0,0 +1,320 @@ +package com.devonfw.tools.solicitor.componentinfo.curation.model; + +import java.util.regex.Pattern; + +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * License finding curation. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class LicenseCuration { + + /** + * Interface data structure for new license data (for ADD or REPLACE). + */ + public static class NewLicenseData { + /** + * new license id + */ + public String license; + + /** + * new url pointing to license text + */ + public String url; + } + + private CurationOperation operation; + + private Pattern path; + + private Pattern ruleIdentifier; + + private Pattern matchedText; + + private Pattern oldLicense; + + private String newLicense; + + private String url; + + private String comment; + + /** + * The constructor. + */ + public LicenseCuration() { + + } + + /** + * @return operation + */ + public CurationOperation getOperation() { + + return this.operation; + } + + /** + * Returns the regular expressing which specifies the file path(s) to which this curation should apply. + * + * @return path + */ + public String getPath() { + + return this.path.toString(); + } + + /** + * Returns the regular expressing which specifies the rule identifiers(s) to which this curation should apply. + * + * @return ruleIdentifier + */ + public String getRuleIdentifier() { + + return this.ruleIdentifier.toString(); + } + + /** + * Returns the regular expressing which specifies the matched text(s) to which this curation should apply. + * + * @return matchedText + */ + public String getMatchedText() { + + return this.matchedText.toString(); + } + + /** + * Returns the regular expressing which specifies the old license (id) to which this curation should apply. + * + * @return new oldLicense + */ + public String getOldLicense() { + + return this.oldLicense.toString(); + } + + /** + * @return new newLicense + */ + public String getNewLicense() { + + return this.newLicense; + } + + /** + * @return url + */ + public String getUrl() { + + return this.url; + } + + /** + * Returns the + * + * @return comment + */ + public String getComment() { + + return this.comment; + } + + /** + * @param operation new value of {@link #getOperation}. + */ + public void setOperation(CurationOperation operation) { + + this.operation = operation; + } + + /** + * Sets the regular expression for specifying the file paths(s) to which this curation should apply. + * + * @param path new value of {@link #getPath}. + */ + public void setPath(String path) { + + this.path = Pattern.compile(path); + } + + /** + * Sets the regular expression for specifying the rule identifier(s) to which this curation should apply + * + * @param ruleIdentifier new value of {@link #getRuleIdentifier}. + */ + public void setRuleIdentifier(String ruleIdentifier) { + + this.ruleIdentifier = Pattern.compile(ruleIdentifier); + } + + /** + * Sets the regular expression for specifying the matched text(s) to which this curation should apply + * + * @param matchedText new value of {@link #getMatchedText}. + */ + public void setMatchedText(String matchedText) { + + this.matchedText = Pattern.compile(matchedText, Pattern.DOTALL); + } + + /** + * Sets the regular expression for specifying the old license (id) to which this curation should apply + * + * @param oldLicense new value of {@link #getOldLicense}. + */ + public void setOldLicense(String oldLicense) { + + this.oldLicense = Pattern.compile(oldLicense, Pattern.DOTALL); + } + + /** + * @param newLicense new value of {@link #getNewLicense}. + */ + public void setNewLicense(String newLicense) { + + this.newLicense = newLicense; + } + + /** + * @param url new value of {@link #getUrl}. + */ + public void setUrl(String url) { + + this.url = url; + } + + /** + * Sets the comment describing the curation + * + * @param comment new value of {@link #getComment}. + */ + public void setComment(String comment) { + + this.comment = comment; + } + + /** + * Check if this is a REMOVE or REPLACE curation and matches the given data. If any of the conditions are not set (via + * the appropriate setters or by reading the data from yaml) then the condition will be ignored. + * + * @param path the path to check + * @param ruleIdentifier the ruleIdentifier to check + * @param matchedText the matchedText to check + * @param oldLicense the old license (as produced by the scancode rule) to check + * @return true if this is a REMOVE or REPLACE curation and the conditions match, false + * otherwise. + */ + @SuppressWarnings("hiding") + public boolean matches(String path, String ruleIdentifier, String matchedText, String oldLicense) { + + if (this.operation != CurationOperation.REMOVE && this.operation != CurationOperation.REPLACE) { + return false; + } + + if (this.path != null && (path == null || !this.path.matcher(path).matches())) { + return false; + } + if (this.ruleIdentifier != null + && (ruleIdentifier == null || !this.ruleIdentifier.matcher(ruleIdentifier).matches())) { + return false; + } + if (this.matchedText != null && (matchedText == null || !this.matchedText.matcher(matchedText).matches())) { + return false; + } + if (this.oldLicense != null && (oldLicense == null || !this.oldLicense.matcher(oldLicense).matches())) { + return false; + } + return true; + + } + + /** + * Check if this is a ADD curation and matches the given path. If any of the conditions are not set (via the appropriate + * setters or by reading the data from yaml) then the condition will be ignored. + * + * @param path the path to check + * @return true if this is a ADD curation and the conditions match, false otherwise. + */ + @SuppressWarnings("hiding") + public boolean matches(String path) { + + if (this.operation != CurationOperation.ADD) { + // no match if it is not an ADD curation + return false; + } + + if (this.path == null) { + // if path condition is not set then this will only match if being called with null argument. + return path == null; + } else { + if (path == null) { + return false; + } + // if path condition is set the check if the given path matches the Pattern + return this.path.matcher(path).matches(); + } + + } + + /** + * Returns the new license data to be applied in case that the curation rule matches. + * + * @return the new license data. null indicates that the found license should be removed (REMOVE + * operation). If the data fields license or url contain null this indicates that they should + * remain unchanged. (Only applicable for REPLACE operation) + */ + public NewLicenseData newLicenseData() { + + if (this.operation == CurationOperation.REMOVE) { + // in case of REMOVE indicate this by returning null + return null; + } + NewLicenseData result = new NewLicenseData(); + result.license = this.newLicense; + result.url = this.url; + return result; + } + + /** + * Validates if the data of this object is consistent. + * + * @throws CurationInvalidException if the object is invalid + */ + public void validate() throws CurationInvalidException { + + if (this.operation == null) { + throw new CurationInvalidException("Operation must not be null for license curation"); + } + if (this.operation == CurationOperation.REMOVE || this.operation == CurationOperation.REPLACE) { + if (this.path == null && this.ruleIdentifier == null && this.matchedText == null && this.oldLicense == null) { + throw new CurationInvalidException( + "For REMOVE/REPLACE license curation at least one condition must be defined"); + } + } + if (this.operation == CurationOperation.REMOVE) { + if (this.newLicense != null || this.url != null) { + throw new CurationInvalidException("For REMOVE license curation neither newLicense nor url must be set"); + } + } + if (this.operation == CurationOperation.REPLACE) { + if (this.newLicense == null && this.url == null) { + throw new CurationInvalidException("For REPLACE license curation at least license or url must be set"); + } + } + if (this.operation == CurationOperation.ADD) { + if (this.ruleIdentifier != null || this.matchedText != null || this.oldLicense != null) { + throw new CurationInvalidException( + "For ADD license curation at neither ruleIdentifier nor matchedText nor oldLicense must be defined"); + } + if (this.newLicense == null || this.url == null) { + throw new CurationInvalidException("For ADD license curation license and url must be set"); + } + } + + } + +} \ No newline at end of file diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProvider.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProvider.java index 8102cdd5..5765b383 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProvider.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProvider.java @@ -17,9 +17,13 @@ import com.devonfw.tools.solicitor.componentinfo.CurationDataHandle; import com.devonfw.tools.solicitor.componentinfo.DataStatusValue; import com.devonfw.tools.solicitor.componentinfo.DefaultComponentInfoImpl; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; import com.devonfw.tools.solicitor.componentinfo.curation.CurationProvider; import com.devonfw.tools.solicitor.componentinfo.curation.FilteredComponentInfoProvider; import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.CurationOperation; +import com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration; import com.devonfw.tools.solicitor.componentinfo.scancode.ScancodeComponentInfo.ScancodeComponentInfoData; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -44,8 +48,6 @@ public class FilteredScancodeComponentInfoProvider implements FilteredComponentI private double licenseToTextRatioToTakeCompleteFile = 90; - private AllKindsPackageURLHandler packageURLHandler; - private ScancodeRawComponentInfoProvider fileScancodeRawComponentInfoProvider; private CurationProvider curationProvider; @@ -63,7 +65,6 @@ public FilteredScancodeComponentInfoProvider(ScancodeRawComponentInfoProvider fi AllKindsPackageURLHandler packageURLHandler, CurationProvider curationProvider) { this.fileScancodeRawComponentInfoProvider = fileScancodeRawComponentInfoProvider; - this.packageURLHandler = packageURLHandler; this.curationProvider = curationProvider; } @@ -98,10 +99,11 @@ public void setMinLicensefileNumberOfLines(int minLicensefileNumberOfLines) { * @return the read scancode information * @throws ComponentInfoAdapterException if there was an exception when reading the data. In case that there is no * data available no exception will be thrown. Instead null will be returned in such a case. + * @throws CurationInvalidException if the curation data is not valid */ @Override public ComponentInfo getComponentInfo(String packageUrl, CurationDataHandle curationDataHandle) - throws ComponentInfoAdapterException { + throws ComponentInfoAdapterException, CurationInvalidException { ScancodeRawComponentInfo rawScancodeData; try { @@ -143,14 +145,15 @@ private void addSupplementedData(ScancodeRawComponentInfo rawScancodeData, * @param curationDataHandle identifies which source should be used for the curation data. * @return the ScancodeComponentInfo * @throws ComponentInfoAdapterException if there was an issue during parsing + * @throws CurationInvalidException if the curation data is not valid */ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, ScancodeRawComponentInfo rawScancodeData, - CurationDataHandle curationDataHandle) throws ComponentInfoAdapterException { + CurationDataHandle curationDataHandle) throws ComponentInfoAdapterException, CurationInvalidException { ScancodeComponentInfo componentScancodeInfos = new ScancodeComponentInfo(this.minLicenseScore, this.minLicensefileNumberOfLines); componentScancodeInfos.setPackageUrl(packageUrl); - // set status to NO_ISSUES. This might be overridden later if issues are detected + // set status to NO_ISSUES. This might be overridden later if issues are detected or curations are applied componentScancodeInfos.setDataStatus(DataStatusValue.NO_ISSUES); // get the object which hold the actual data @@ -161,8 +164,12 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod // Get all excludedPaths in this curation List excludedPaths = null; + List licenseCurations = null; + List copyrightCurations = null; if (componentInfoCuration != null) { excludedPaths = componentInfoCuration.getExcludedPaths(); + licenseCurations = componentInfoCuration.getLicenseCurations(); + copyrightCurations = componentInfoCuration.getCopyrightCurations(); } JsonNode scancodeJson; @@ -176,6 +183,8 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod for (JsonNode file : scancodeJson.get("files")) { String path = file.get("path").asText(); if (isExcluded(path, excludedPaths)) { + // this is a curation operation, so set the status + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); continue; } if ("directory".equals(file.get("type").asText())) { @@ -188,10 +197,25 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod double licenseTextRatio = file.get("percentage_of_license_text").asDouble(); boolean takeCompleteFile = licenseTextRatio >= this.licenseToTextRatioToTakeCompleteFile; for (JsonNode cr : file.get("copyrights")) { + String copyright; if (cr.has("copyright")) { - scancodeComponentInfoData.addCopyright(cr.get("copyright").asText()); + copyright = cr.get("copyright").asText(); } else { - scancodeComponentInfoData.addCopyright(cr.get("value").asText()); + copyright = cr.get("value").asText(); + } + String copyrightAfterCuration = getEffectiveCopyrightWithCuration(path, copyright, copyrightCurations); + if (copyrightAfterCuration != null) { + if (!copyrightAfterCuration.equals(copyright)) { + // the copyright info changed due to applying a curation, so set the status + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); + } + scancodeComponentInfoData.addCopyright(copyrightAfterCuration); + } else { + if (copyright != null) { + // the copyright info was removed due to applying a curation, so set the status + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); + + } } } @@ -200,7 +224,13 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod boolean classPathExceptionExists = false; int numberOfGplLicenses = 0; for (JsonNode li : file.get("licenses")) { - String licenseName = li.get("spdx_license_key").asText(); + LicenseCuration.NewLicenseData effective = getEffectiveLicenseInfoWithCuration(path, li, licenseCurations); + if (effective == null) { + // license finding to be REMOVED via finding + continue; + } + String licenseName = effective.license != null ? effective.license : li.get("spdx_license_key").asText(); + if ("Classpath-exception-2.0".equals(licenseName)) { classPathExceptionExists = true; } @@ -228,19 +258,29 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod } } for (JsonNode li : file.get("licenses")) { - String licenseid = li.get("key").asText(); - String licenseName = li.get("spdx_license_key").asText(); + LicenseCuration.NewLicenseData effective = getEffectiveLicenseInfoWithCuration(path, li, licenseCurations); + if (effective == null) { + // license finding to be REMOVED via finding + // this is a curation operation, so set the status + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); + continue; + } + if (effective.license != null || effective.url != null) { + // license or url are altered due to curation, so set the status + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); + } + String licenseName = effective.license != null ? effective.license : li.get("spdx_license_key").asText(); String effectiveLicenseName = spdxIdMap.get(licenseName); if (effectiveLicenseName == null) { // not contained in map --> this must be the Classpath-exception-2.0 continue; } else { licenseName = effectiveLicenseName; - if (licenseName.endsWith("WITH Classpath-exception-2.0")) { - licenseid = licenseid + "WITH Classpath-exception-2.0"; - } } String licenseDefaultUrl = li.get("scancode_text_url").asText(); + if (effective.url != null) { + licenseDefaultUrl = effective.url; + } licenseDefaultUrl = normalizeLicenseUrl(packageUrl, licenseDefaultUrl); double score = li.get("score").asDouble(); String licenseUrl = path; @@ -252,6 +292,13 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod licenseUrl += "-L" + endLine; } } + if (effective.url != null) { + // curation redefined the license URL + licenseUrl = effective.url; + // enforce that the filescore always exceeds the threshold + startLine = 0; + endLine = Integer.MAX_VALUE; + } licenseUrl = normalizeLicenseUrl(packageUrl, licenseUrl); String givenLicenseText = null; @@ -259,10 +306,18 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod givenLicenseText = this.fileScancodeRawComponentInfoProvider.retrieveContent(packageUrl, licenseUrl); } - scancodeComponentInfoData.addLicense(licenseid, licenseName, licenseDefaultUrl, score, licenseUrl, + scancodeComponentInfoData.addLicense(licenseName, licenseName, licenseDefaultUrl, score, licenseUrl, givenLicenseText, endLine - startLine); } + // do any per scanned file postprocessing + addCopyrightsByCuration(path, copyrightCurations, componentScancodeInfos); + addLicensesByCuration(packageUrl, path, licenseCurations, componentScancodeInfos); + } + // add copyrights / licenses due to curations on package level + addCopyrightsByCuration(null, copyrightCurations, componentScancodeInfos); + addLicensesByCuration(packageUrl, null, licenseCurations, componentScancodeInfos); + if (scancodeComponentInfoData.getNoticeFileUrl() != null) { scancodeComponentInfoData.setNoticeFileContent(this.fileScancodeRawComponentInfoProvider .retrieveContent(packageUrl, scancodeComponentInfoData.getNoticeFileUrl())); @@ -270,6 +325,155 @@ private ScancodeComponentInfo parseAndMapScancodeJson(String packageUrl, Scancod return componentScancodeInfos; } + /** + * Gets the effective license info after possibly applying curations for a single license finding. + * + * @param path + * @param li + * @param licenseCurations + * @return + */ + private LicenseCuration.NewLicenseData getEffectiveLicenseInfoWithCuration(String path, JsonNode li, + List licenseCurations) { + + if (licenseCurations == null) { + // NewLicenseData with all members being null indicates: no change + return new LicenseCuration.NewLicenseData(); + } + + String ruleIdentifier = li.get("matched_rule").get("identifier").asText(); + String matchedText = li.get("matched_text").asText(); + String spdxId = li.get("spdx_license_key").asText(); + + for (LicenseCuration rule : licenseCurations) { + if (rule.matches(path, ruleIdentifier, matchedText, spdxId)) { + LicenseCuration.NewLicenseData result = rule.newLicenseData(); + if (LOG.isDebugEnabled()) { + if (result == null) { + LOG.debug("License finding of rule '{}' in '{}' will be ignored due to remove license curation", + ruleIdentifier, path); + } else { + LOG.debug("License finding of rule '{}' in '{}' will be replaced by SPDX-ID '{}' and URL '{}'", + ruleIdentifier, path, result.license, result.url); + + } + } + return result; + } + } + // NewLicenseData with all members being null indicates: no change + return new LicenseCuration.NewLicenseData(); + } + + /** + * Adds license entries due to curations. + * + * @param packageUrl + * @param path + * @param licenseCurations + * @param componentScancodeInfos + */ + private void addLicensesByCuration(String packageUrl, String path, List licenseCurations, + ScancodeComponentInfo componentScancodeInfos) { + + if (licenseCurations == null) { + // no curations available: return empty collection + return; + } + + for (LicenseCuration rule : licenseCurations) { + if (rule.matches(path)) { + if (rule.getOperation() == CurationOperation.ADD) { + LicenseCuration.NewLicenseData license = rule.newLicenseData(); + if (LOG.isDebugEnabled()) { + LOG.debug( + "License finding with SPDX-ID '{}' and url '{}' in '{}' will be added due to ADD copyright curation", + license.license, license.url, path); + } + String licenseUrl = normalizeLicenseUrl(packageUrl, license.url); + String givenLicenseText = this.fileScancodeRawComponentInfoProvider.retrieveContent(packageUrl, licenseUrl); + + componentScancodeInfos.getComponentInfoData().addLicense(license.license, license.license, license.url, 100, + licenseUrl, givenLicenseText, Integer.MAX_VALUE); + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); + } else { + throw new IllegalStateException("This seems to be a bug"); + } + } + } + } + + /** + * Gets the effective copyright after possibly applying curations for a single copyright finding. + * + * @param path + * @param copyright + * @param copyrightCurations + * @return + */ + private String getEffectiveCopyrightWithCuration(String path, String copyright, + List copyrightCurations) { + + if (copyrightCurations == null) { + // no curations available: return the original copyright + return copyright; + } + + for (CopyrightCuration rule : copyrightCurations) { + if (rule.matches(path, copyright)) { + if (rule.getOperation() == CurationOperation.REMOVE) { + if (LOG.isDebugEnabled()) { + LOG.debug("Copyright finding '{}' in '{}' will be ignored due to remove copyright curation", copyright, + path); + } + return null; + } + if (rule.getOperation() == CurationOperation.REPLACE) { + if (LOG.isDebugEnabled()) { + LOG.debug("Copyright finding '{}' in '{}' will be ignored due to remove copyright curation", copyright, + path); + } + return rule.getNewCopyright(); + } + throw new IllegalStateException("This seems to be a bug"); + } + } + // no curations applied: return the original copyright + return copyright; + } + + /** + * Adds copyrights entries due to curations. + * + * @param path + * @param copyrightCurations + * @param componentScancodeInfos + */ + private void addCopyrightsByCuration(String path, List copyrightCurations, + ScancodeComponentInfo componentScancodeInfos) { + + if (copyrightCurations == null) { + // no curations available: return empty collection + return; + } + + for (CopyrightCuration rule : copyrightCurations) { + if (rule.matches(path)) { + if (rule.getOperation() == CurationOperation.ADD) { + String copyrightToBeAdded = rule.getNewCopyright(); + if (LOG.isDebugEnabled()) { + LOG.debug("Copyright finding '{}' in '{}' will be added due to ADD copyright curation", copyrightToBeAdded, + path); + } + componentScancodeInfos.getComponentInfoData().addCopyright(copyrightToBeAdded); + componentScancodeInfos.setDataStatus(DataStatusValue.CURATED); + } else { + throw new IllegalStateException("This seems to be a bug"); + } + } + } + } + /** * Adjustment of license paths/urls so that they might retrieved * diff --git a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfo.java b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfo.java index c8b8923f..23de13e6 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfo.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfo.java @@ -271,7 +271,8 @@ public void addLicense(String licenseId, String licenseName, String licenseDefau String resultingFilePath = existingLicenseInfo.getLicenseUrl(); String resultingGivenText = existingLicenseInfo.getGivenLicenseText(); int resultingFileScore = existingLicenseInfo.getLicenseFileScore(); - if (fileScore > existingLicenseInfo.getLicenseFileScore()) { + if (fileScore > existingLicenseInfo.getLicenseFileScore() && givenLicenseText != null + && !givenLicenseText.isEmpty()) { resultingFilePath = filePath; resultingFileScore = fileScore; resultingGivenText = givenLicenseText; @@ -281,8 +282,10 @@ public void addLicense(String licenseId, String licenseName, String licenseDefau } else { if (score >= this.minLicenseScore) { - this.licenses.put(licenseId, new ScancodeLicenseInfo(licenseId, licenseName, licenseDefaultUrl, score, - filePath, givenLicenseText, fileScore, this.minLicensefileNumberOfLines)); + this.licenses.put(licenseId, + new ScancodeLicenseInfo(licenseId, licenseName, licenseDefaultUrl, score, filePath, givenLicenseText, + (givenLicenseText != null && !givenLicenseText.isEmpty()) ? fileScore : 0, + this.minLicensefileNumberOfLines)); } } } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractHierarchicalCurationProviderTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractHierarchicalCurationProviderTest.java new file mode 100644 index 00000000..e78481d5 --- /dev/null +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/AbstractHierarchicalCurationProviderTest.java @@ -0,0 +1,191 @@ +package com.devonfw.tools.solicitor.componentinfo.curation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.devonfw.tools.solicitor.common.packageurl.AllKindsPackageURLHandler; +import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; +import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; +import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.CurationOperation; + +/** + * Tests for {@link AbstractHierarchicalCurationProvider}. + * + */ +class AbstractHierarchicalCurationProviderTest { + + @Test + void testFindCurationsHierarchyOnDefaultCurationDataSelector() + throws ComponentInfoAdapterNonExistingCurationDataSelectorException, ComponentInfoAdapterException, + CurationInvalidException { + + AbstractHierarchicalCurationProvider provider = createObjectUnderTest(true); + + ComponentInfoCuration result = provider.findCurations("pkg:/maven/ch.qos.logback/logback-classic@1.2.3", + new SelectorCurationDataHandle(null)); + assertEquals("pkg/maven/ch/qos/logback/logback-classic/1.2.3", result.getName()); + assertEquals(7, result.getCopyrightCurations().size()); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright().contains("theDefault")); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright() + .endsWith("pkg/maven/ch/qos/logback/logback-classic/1.2.3")); + assertTrue(result.getCopyrightCurations().get(6).getNewCopyright().endsWith("pkg")); + + } + + @Test + void testFindCurationsHierarchyOnNonDefaultCurationDataSelector() + throws ComponentInfoAdapterNonExistingCurationDataSelectorException, ComponentInfoAdapterException, + CurationInvalidException { + + AbstractHierarchicalCurationProvider provider = createObjectUnderTest(true); + + ComponentInfoCuration result = provider.findCurations("pkg:/maven/ch.qos.logback/logback-classic@1.2.3", + new SelectorCurationDataHandle("someSelector")); + assertEquals("pkg/maven/ch/qos/logback/logback-classic/1.2.3", result.getName()); + assertEquals(7, result.getCopyrightCurations().size()); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright().contains("someSelector")); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright() + .endsWith("pkg/maven/ch/qos/logback/logback-classic/1.2.3")); + assertTrue(result.getCopyrightCurations().get(6).getNewCopyright().endsWith("pkg")); + + } + + @Test + void testFindCurationsHierarchyWhenCurationExistOnlyForVersion() + throws ComponentInfoAdapterNonExistingCurationDataSelectorException, ComponentInfoAdapterException, + CurationInvalidException { + + AbstractHierarchicalCurationProvider provider = createObjectUnderTest(true); + + ComponentInfoCuration result = provider.findCurations("pkg:/maven/ch.qos.logback/logback-classic@1.2.3", + new SelectorCurationDataHandle("curationonlyonversion")); + assertEquals("pkg/maven/ch/qos/logback/logback-classic/1.2.3", result.getName()); + assertEquals(1, result.getCopyrightCurations().size()); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright().contains("curationonlyonversion")); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright() + .endsWith("pkg/maven/ch/qos/logback/logback-classic/1.2.3")); + } + + @Test + void testFindCurationsHierarchyDisabled() throws ComponentInfoAdapterNonExistingCurationDataSelectorException, + ComponentInfoAdapterException, CurationInvalidException { + + AbstractHierarchicalCurationProvider provider = createObjectUnderTest(false); + + ComponentInfoCuration result = provider.findCurations("pkg:/maven/ch.qos.logback/logback-classic@1.2.3", + new SelectorCurationDataHandle(null)); + assertEquals("pkg/maven/ch/qos/logback/logback-classic/1.2.3", result.getName()); + assertEquals(1, result.getCopyrightCurations().size()); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright().contains("theDefault")); + assertTrue(result.getCopyrightCurations().get(0).getNewCopyright() + .endsWith("pkg/maven/ch/qos/logback/logback-classic/1.2.3")); + } + + @Test + void testFindCurationsInvalidSelector() throws ComponentInfoAdapterNonExistingCurationDataSelectorException, + ComponentInfoAdapterException, CurationInvalidException { + + AbstractHierarchicalCurationProvider provider = createObjectUnderTest(true); + + ComponentInfoAdapterNonExistingCurationDataSelectorException e = assertThrows( + ComponentInfoAdapterNonExistingCurationDataSelectorException.class, + () -> provider.findCurations("pkg:/maven/ch.qos.logback/logback-classic@1.2.3", + new SelectorCurationDataHandle("invalid"))); + assertEquals("test1", e.getMessage()); + } + + @Test + void testFindCurationsNonexistentSelector() throws ComponentInfoAdapterNonExistingCurationDataSelectorException, + ComponentInfoAdapterException, CurationInvalidException { + + AbstractHierarchicalCurationProvider provider = createObjectUnderTest(true); + + ComponentInfoAdapterNonExistingCurationDataSelectorException e = assertThrows( + ComponentInfoAdapterNonExistingCurationDataSelectorException.class, + () -> provider.findCurations("pkg:/maven/ch.qos.logback/logback-classic@1.2.3", + new SelectorCurationDataHandle("nonexistent"))); + assertEquals("test2", e.getMessage()); + } + + /** + * Creates a instance of the {@link AbstractHierarchicalCurationProvider}. Abstract methods will be implemented to + * allow for testing. + * + * @return the object to be tested. + */ + private AbstractHierarchicalCurationProvider createObjectUnderTest(boolean evaluateHierarchy) { + + AllKindsPackageURLHandler packageUrlHandler = Mockito.mock(AllKindsPackageURLHandler.class); + Mockito.when(packageUrlHandler.pathFor("pkg:/maven/ch.qos.logback/logback-classic@1.2.3")) + .thenReturn("pkg/maven/ch/qos/logback/logback-classic/1.2.3"); + + AbstractHierarchicalCurationProvider provider = new AbstractHierarchicalCurationProvider(packageUrlHandler) { + + @Override + protected void validateEffectiveCurationDataSelector(String effectiveCurationDataSelector) + throws ComponentInfoAdapterNonExistingCurationDataSelectorException { + + if (effectiveCurationDataSelector.equals("invalid")) { + throw new ComponentInfoAdapterNonExistingCurationDataSelectorException("test1"); + } + + } + + @Override + protected boolean isHierarchyEvaluation() { + + return evaluateHierarchy; + } + + @Override + protected ComponentInfoCuration fetchCurationFromRepository(String effectiveCurationDataSelector, + String pathFragmentWithinRepo) throws ComponentInfoAdapterException, CurationInvalidException { + + if (effectiveCurationDataSelector.equals("nonexistent") || effectiveCurationDataSelector.equals("empty")) { + return null; + } + if (effectiveCurationDataSelector.equals("curationonlyonversion") + && !pathFragmentWithinRepo.equals("pkg/maven/ch/qos/logback/logback-classic/1.2.3")) { + return null; + } + ComponentInfoCuration cic = new ComponentInfoCuration(); + cic.setName(pathFragmentWithinRepo); + CopyrightCuration cc = new CopyrightCuration(); + cc.setOperation(CurationOperation.ADD); + cc.setNewCopyright("Copyright " + effectiveCurationDataSelector + " " + pathFragmentWithinRepo); + cic.setCopyrightCurations(List.of(cc)); + return cic; + } + + @Override + protected String determineEffectiveCurationDataSelector(String curationDataSelector) { + + if (curationDataSelector == null) { + return "theDefault"; + } else { + return curationDataSelector; + } + } + + @Override + protected void assureCurationDataSelectorAvailable(String effectiveCurationDataSelector) + throws ComponentInfoAdapterException, ComponentInfoAdapterNonExistingCurationDataSelectorException { + + if (effectiveCurationDataSelector.equals("nonexistent")) { + throw new ComponentInfoAdapterNonExistingCurationDataSelectorException("test2"); + } + + } + }; + return provider; + } + +} diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curating/CurationReadingTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationReadingTest.java similarity index 58% rename from core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curating/CurationReadingTest.java rename to core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationReadingTest.java index f7841fd8..ca294d6f 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curating/CurationReadingTest.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationReadingTest.java @@ -1,4 +1,4 @@ -package com.devonfw.tools.solicitor.componentinfo.curating; +package com.devonfw.tools.solicitor.componentinfo.curation; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -11,6 +11,7 @@ import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; import com.devonfw.tools.solicitor.componentinfo.curation.model.CurationList; +import com.devonfw.tools.solicitor.componentinfo.curation.model.CurationOperation; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -23,10 +24,11 @@ class CurationReadingTest { * Test reading a curation for a component which has all fields set. * * @throws IOException might be thown by the ObjectMapper. + * @throws CurationInvalidException * */ @Test - public void testReadSingleCurationComplete() throws IOException { + public void testReadSingleCurationComplete() throws IOException, CurationInvalidException { // given ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); @@ -47,6 +49,34 @@ public void testReadSingleCurationComplete() throws IOException { assertEquals(2, curation.getCopyrights().size()); assertEquals("Copyright (c) 2003 First Holder", curation.getCopyrights().get(0)); + assertEquals(4, curation.getLicenseCurations().size()); + curation.getLicenseCurations().get(0).validate(); + assertEquals("some/path/1", curation.getLicenseCurations().get(0).getPath()); + assertEquals(CurationOperation.REMOVE, curation.getLicenseCurations().get(0).getOperation()); + curation.getLicenseCurations().get(1).validate(); + assertEquals("some/path/2", curation.getLicenseCurations().get(1).getPath()); + assertEquals(CurationOperation.REMOVE, curation.getLicenseCurations().get(1).getOperation()); + curation.getLicenseCurations().get(2).validate(); + assertEquals("another/path/1", curation.getLicenseCurations().get(2).getPath()); + assertEquals(CurationOperation.ADD, curation.getLicenseCurations().get(2).getOperation()); + curation.getLicenseCurations().get(3).validate(); + assertEquals("another/path/2", curation.getLicenseCurations().get(3).getPath()); + assertEquals(CurationOperation.ADD, curation.getLicenseCurations().get(3).getOperation()); + + assertEquals(4, curation.getCopyrightCurations().size()); + curation.getCopyrightCurations().get(0).validate(); + assertEquals("other/path/1", curation.getCopyrightCurations().get(0).getPath()); + assertEquals(CurationOperation.REMOVE, curation.getCopyrightCurations().get(0).getOperation()); + curation.getCopyrightCurations().get(1).validate(); + assertEquals("other/path/2", curation.getCopyrightCurations().get(1).getPath()); + assertEquals(CurationOperation.REMOVE, curation.getCopyrightCurations().get(1).getOperation()); + curation.getCopyrightCurations().get(2).validate(); + assertEquals("some/path/1", curation.getCopyrightCurations().get(2).getPath()); + assertEquals(CurationOperation.ADD, curation.getCopyrightCurations().get(2).getOperation()); + curation.getCopyrightCurations().get(3).validate(); + assertEquals("some/path/2", curation.getCopyrightCurations().get(3).getPath()); + assertEquals(CurationOperation.ADD, curation.getCopyrightCurations().get(3).getOperation()); + } /** @@ -77,7 +107,7 @@ public void testReadSingleCurationPartial() throws IOException { } /** - * Test reading an aarry of curation infos. + * Test reading an array of curation infos. * * @throws IOException might be thown by the ObjectMapper. * @@ -95,11 +125,13 @@ public void testReadListOfCurations() throws IOException { // then assertNotNull(curations); - assertEquals(2, curations.getArtifacts().size()); + assertEquals(4, curations.getArtifacts().size()); assertEquals("pkg/maven/somenamespace/somecomponent/2.3.4", curations.getArtifacts().get(0).getName()); assertEquals("Apache-2.0", curations.getArtifacts().get(0).getLicenses().get(0).getLicense()); assertEquals("pkg/maven/somenamespace/somecomponent/2.3.5", curations.getArtifacts().get(1).getName()); assertEquals("BSD-2-Clause", curations.getArtifacts().get(1).getLicenses().get(0).getLicense()); + assertEquals("pkg/maven/othernamespace", curations.getArtifacts().get(2).getName()); + assertEquals("pkg/maven/othernamespace/othercomponent", curations.getArtifacts().get(3).getName()); } } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationUtilTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationUtilTest.java new file mode 100644 index 00000000..efa4b193 --- /dev/null +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/CurationUtilTest.java @@ -0,0 +1,101 @@ +package com.devonfw.tools.solicitor.componentinfo.curation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.devonfw.tools.solicitor.common.IOHelper; +import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration; +import com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseInfoCuration; + +/** + * Test for {@link CurationUtil}. + * + */ +class CurationUtilTest { + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.CurationUtil#curationDataFromString(java.lang.String)}. + * + * @throws CurationInvalidException + * @throws IOException + */ + @Test + void testCurationDataFromString() throws CurationInvalidException, IOException { + + try (FileInputStream is = new FileInputStream("src/test/resources/curations/curations.txt")) { + + String testData = IOHelper.readStringFromInputStream(is); + ComponentInfoCuration curations = CurationUtil.curationDataFromString(testData); + assertEquals("some note", curations.getNote()); + } + } + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.CurationUtil#curationDataFromString(java.lang.String)}. + * + * @throws CurationInvalidException + * @throws IOException + */ + @Test + void testCurationDataFromStringMissingName() throws CurationInvalidException, IOException { + + assertThrows(CurationInvalidException.class, () -> CurationUtil.curationDataFromString("foo")); + } + + /** + * Test method for {@link CurationUtil#merge(ComponentInfoCuration, ComponentInfoCuration)} + */ + @Test + void testMerge() { + + ComponentInfoCuration curationFirst = new ComponentInfoCuration(); + curationFirst.setName("firstName"); + curationFirst.setNote("firstNote"); + curationFirst.setUrl("firstUrl"); + curationFirst.setExcludedPaths(List.of("first path")); + curationFirst.setCopyrights(List.of("first copyright 1", "first copyright 2")); + LicenseInfoCuration licFirst = new LicenseInfoCuration(); + curationFirst.setLicenses(List.of(licFirst)); + CopyrightCuration ccFirst = new CopyrightCuration(); + curationFirst.setCopyrightCurations(List.of(ccFirst)); + LicenseCuration lcFirst = new LicenseCuration(); + curationFirst.setLicenseCurations(List.of(lcFirst)); + + ComponentInfoCuration curationSecond = new ComponentInfoCuration(); + curationSecond.setName("secondName"); + curationSecond.setNote("secondNote"); + curationSecond.setUrl("secondUrl"); + curationSecond.setExcludedPaths(List.of("second path")); + curationSecond.setCopyrights(List.of("second copyright 1", "second copyright 2")); + LicenseInfoCuration licSecond = new LicenseInfoCuration(); + curationSecond.setLicenses(List.of(licSecond)); + CopyrightCuration ccSecond = new CopyrightCuration(); + curationSecond.setCopyrightCurations(List.of(ccSecond)); + LicenseCuration lcSecond = new LicenseCuration(); + curationSecond.setLicenseCurations(List.of(lcSecond)); + + ComponentInfoCuration merged = CurationUtil.merge(curationFirst, curationSecond); + assertEquals("secondName", merged.getName()); + assertTrue(merged.getNote().startsWith("secondNote")); + assertTrue(merged.getNote().contains("/")); + assertTrue(merged.getNote().endsWith("firstNote")); + assertEquals("secondUrl", merged.getUrl()); + assertTrue(merged.getCopyrights() + .equals(List.of("second copyright 1", "second copyright 2", "first copyright 1", "first copyright 2"))); + assertTrue(merged.getLicenses().equals(List.of(licSecond, licFirst))); + assertTrue(merged.getExcludedPaths().equals(List.of("second path", "first path"))); + assertTrue(merged.getLicenseCurations().equals(List.of(lcSecond, lcFirst))); + assertTrue(merged.getCopyrightCurations().equals(List.of(ccSecond, ccFirst))); + } +} diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curating/SingleFileCurationProviderTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProviderTest.java similarity index 71% rename from core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curating/SingleFileCurationProviderTest.java rename to core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProviderTest.java index e95082ac..00970f81 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curating/SingleFileCurationProviderTest.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/SingleFileCurationProviderTest.java @@ -1,4 +1,4 @@ -package com.devonfw.tools.solicitor.componentinfo.curating; +package com.devonfw.tools.solicitor.componentinfo.curation; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -10,7 +10,6 @@ import com.devonfw.tools.solicitor.common.packageurl.AllKindsPackageURLHandler; import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; -import com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider; import com.devonfw.tools.solicitor.componentinfo.curation.model.ComponentInfoCuration; /** @@ -29,6 +28,8 @@ public void setup() { .thenReturn("pkg/maven/somenamespace/somecomponent/2.3.4"); Mockito.when(packageUrlHandler.pathFor("pkg:maven/somenamespace/somecomponent@2.3.5")) .thenReturn("pkg/maven/somenamespace/somecomponent/2.3.5"); + Mockito.when(packageUrlHandler.pathFor("pkg:maven/othernamespace/othercomponent@1.2.3")) + .thenReturn("pkg/maven/othernamespace/othercomponent/1.2.3"); this.objectUnderTest = new SingleFileCurationProvider(packageUrlHandler); this.objectUnderTest.setCurationsFileName("src/test/resources/curations/array_of_curations.yaml"); @@ -39,9 +40,10 @@ public void setup() { * {@link com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider#findCurations(java.lang.String, java.lang.String)}. * * @throws ComponentInfoAdapterException + * @throws CurationInvalidException */ @Test - void testFindCurationsWithSelectorNull() throws ComponentInfoAdapterException { + void testFindCurationsWithSelectorNull() throws ComponentInfoAdapterException, CurationInvalidException { ComponentInfoCuration result; @@ -59,9 +61,10 @@ void testFindCurationsWithSelectorNull() throws ComponentInfoAdapterException { * {@link com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider#findCurations(java.lang.String, java.lang.String)}. * * @throws ComponentInfoAdapterException + * @throws CurationInvalidException */ @Test - void testFindCurationsWithSelectorNone() throws ComponentInfoAdapterException { + void testFindCurationsWithSelectorNone() throws ComponentInfoAdapterException, CurationInvalidException { ComponentInfoCuration result; @@ -69,4 +72,23 @@ void testFindCurationsWithSelectorNone() throws ComponentInfoAdapterException { new SelectorCurationDataHandle("none")); assertNull(result); } + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider#findCurations(java.lang.String, java.lang.String)}. + * + * @throws ComponentInfoAdapterException + * @throws CurationInvalidException + */ + @Test + void testFindCurationsUsingHierarchy() throws ComponentInfoAdapterException, CurationInvalidException { + + ComponentInfoCuration result; + + result = this.objectUnderTest.findCurations("pkg:maven/othernamespace/othercomponent@1.2.3", + new SelectorCurationDataHandle(null)); + assertEquals("some/path/2", result.getLicenseCurations().get(0).getPath()); + assertEquals("some/path/1", result.getLicenseCurations().get(1).getPath()); + } + } diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CopyrightCurationTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CopyrightCurationTest.java new file mode 100644 index 00000000..6151a1fc --- /dev/null +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/model/CopyrightCurationTest.java @@ -0,0 +1,162 @@ +package com.devonfw.tools.solicitor.componentinfo.curation.model; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; + +/** + * Tests for {@link CopyrightCuration}. Note that not all possible data constellations are tested but just the main + * behavior. + * + */ +class CopyrightCurationTest { + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration#matches(java.lang.String, java.lang.String)}. + */ + @Test + void testMatchesStringString() { + + CopyrightCuration cc = new CopyrightCuration(); + cc.setOperation(CurationOperation.ADD); + + assertFalse(cc.matches(null, null)); + + cc.setOperation(CurationOperation.REMOVE); + assertTrue(cc.matches(null, null)); + + cc.setPath(".*abc.*"); + cc.setOldCopyright(".*456.*"); + assertTrue(cc.matches("abcd", "4567")); + assertFalse(cc.matches("bcd", "4567")); + assertFalse(cc.matches("abcd", "567")); + } + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration#matches(java.lang.String)}. + */ + @Test + void testMatchesString() { + + CopyrightCuration cc = new CopyrightCuration(); + cc.setOperation(CurationOperation.REMOVE); + + assertFalse(cc.matches(null)); + + cc.setOperation(CurationOperation.ADD); + assertTrue(cc.matches(null)); + assertFalse(cc.matches("foo")); + + cc.setPath(".*abc.*"); + assertTrue(cc.matches("abcd")); + assertFalse(cc.matches("bcd")); + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateNoOp() throws CurationInvalidException { + + // no operation defined + CopyrightCuration cc = new CopyrightCuration(); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "Operation must not be null for copyright curation"); + + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateAdd() throws CurationInvalidException { + + CopyrightCuration cc = new CopyrightCuration(); + + cc.setOperation(CurationOperation.ADD); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "For ADD copyright curation newCopyright must be set"); + + cc.setNewCopyright("foo"); + + cc.validate(); // should be ok + + cc.setOldCopyright("bar"); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "For ADD copyright curation oldCopyright must not be defined"); + + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateRemove() throws CurationInvalidException { + + CopyrightCuration cc = new CopyrightCuration(); + + cc.setOperation(CurationOperation.REMOVE); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "For REMOVE/REPLACE copyright curation at least one condition must be defined"); + + cc.setPath("foo"); + + cc.validate(); // should be ok + + cc.setNewCopyright("bar"); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "For REMOVE copyright curation the newCopyright must not be set"); + + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.CopyrightCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateReplace() throws CurationInvalidException { + + CopyrightCuration cc = new CopyrightCuration(); + + cc.setOperation(CurationOperation.REPLACE); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "For REMOVE/REPLACE copyright curation at least one condition must be defined"); + + cc.setPath("foo"); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + cc.validate(); + }, "For REPLACE copyright curation newCopyright must be set"); + + cc.setNewCopyright("bar"); + + cc.validate(); // should be ok + + } + +} diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/model/LicenseCurationTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/model/LicenseCurationTest.java new file mode 100644 index 00000000..0e2c6973 --- /dev/null +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/curation/model/LicenseCurationTest.java @@ -0,0 +1,198 @@ +package com.devonfw.tools.solicitor.componentinfo.curation.model; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; +import com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration.NewLicenseData; + +/** + * Tests for {@link LicenseCuration}. Note that not all possible data constellations are tested but just the main + * behavior. + * + */ +class LicenseCurationTest { + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#matches(java.lang.String, java.lang.String, java.lang.String, java.lang.String)}. + */ + @Test + void testMatchesStringStringStringString() { + + LicenseCuration lc = new LicenseCuration(); + lc.setOperation(CurationOperation.ADD); + + assertFalse(lc.matches(null, null, null, null)); + + lc.setOperation(CurationOperation.REMOVE); + assertTrue(lc.matches(null, null, null, null)); + + lc.setPath(".*abc.*"); + lc.setRuleIdentifier(".*def.*"); + lc.setMatchedText(".*123.*"); + lc.setOldLicense(".*456.*"); + assertTrue(lc.matches("abcd", "cdef", "z123", "4567")); + assertFalse(lc.matches("bcd", "cdef", "z123", "4567")); + assertFalse(lc.matches("abcd", "cde", "z123", "4567")); + assertFalse(lc.matches("abcd", "cdef", "z12", "4567")); + assertFalse(lc.matches("abcd", "cdef", "z123", "567")); + + } + + /** + * Test method for + * {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#matches(java.lang.String)}. + */ + @Test + void testMatchesString() { + + LicenseCuration lc = new LicenseCuration(); + lc.setOperation(CurationOperation.REMOVE); + + assertFalse(lc.matches(null)); + + lc.setOperation(CurationOperation.ADD); + assertTrue(lc.matches(null)); + assertFalse(lc.matches("foo")); + + lc.setPath(".*abc.*"); + assertTrue(lc.matches("abcd")); + assertFalse(lc.matches("bcd")); + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#newLicenseData()}. + */ + @Test + void testNewLicenseData() { + + LicenseCuration lc = new LicenseCuration(); + lc.setOperation(CurationOperation.REMOVE); + + assertNull(lc.newLicenseData()); + + lc.setOperation(CurationOperation.ADD); + NewLicenseData nld = lc.newLicenseData(); + + assertNull(nld.license); + assertNull(nld.url); + + lc.setNewLicense("foo"); + lc.setUrl("bar"); + + nld = lc.newLicenseData(); + + assertEquals("foo", nld.license); + assertEquals("bar", nld.url); + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateNoOp() throws CurationInvalidException { + + // no operation defined + LicenseCuration lc = new LicenseCuration(); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "Operation must not be null for license curation"); + + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateAdd() throws CurationInvalidException { + + LicenseCuration lc = new LicenseCuration(); + + lc.setOperation(CurationOperation.ADD); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For ADD license curation license and url must be set"); + + lc.setNewLicense("foo"); + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For ADD license curation license and url must be set"); + + lc.setUrl("bar"); + lc.validate(); // should be ok + + lc.setMatchedText("some text"); + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For ADD license curation at neither ruleIdentifier nor matchedText nor oldLicense must be defined"); + + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateRemove() throws CurationInvalidException { + + LicenseCuration lc = new LicenseCuration(); + + lc.setOperation(CurationOperation.REMOVE); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For REMOVE/REPLACE license curation at least one condition must be defined"); + + lc.setMatchedText("some text"); + + lc.validate(); // should be ok + + lc.setNewLicense("foo"); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For REMOVE license curation neither newLicense nor url must be set"); + + } + + /** + * Test method for {@link com.devonfw.tools.solicitor.componentinfo.curation.model.LicenseCuration#validate()}. + * + * @throws CurationInvalidException if the curation is not valid + */ + @Test + void testValidateReplace() throws CurationInvalidException { + + LicenseCuration lc = new LicenseCuration(); + + lc.setOperation(CurationOperation.REPLACE); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For REMOVE/REPLACE license curation at least one condition must be defined"); + + lc.setMatchedText("some text"); + + Assertions.assertThrows(CurationInvalidException.class, () -> { + lc.validate(); + }, "For REPLACE license curation at least license or url must be set"); + + lc.setNewLicense("foo"); + + lc.validate(); // should be ok + + } +} diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProviderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProviderTests.java index 0d5cafdb..5e95bb00 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProviderTests.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/FilteredScancodeComponentInfoProviderTests.java @@ -11,6 +11,7 @@ import com.devonfw.tools.solicitor.componentinfo.ComponentInfo; import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; import com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider; /** @@ -49,9 +50,10 @@ public void setup() { * file exists * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException if the curation data is not valid */ @Test - public void testGetComponentInfoWithoutCurations() throws ComponentInfoAdapterException { + public void testGetComponentInfoWithoutCurations() throws ComponentInfoAdapterException, CurationInvalidException { // given this.singleFileCurationProvider.setCurationsFileName("src/test/resources/scancodefileadapter/nonexisting.yaml"); @@ -76,9 +78,11 @@ public void testGetComponentInfoWithoutCurations() throws ComponentInfoAdapterEx * excluded * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException if the curation data is not valid */ @Test - public void testGetComponentInfoWithCurationsAndExclusions() throws ComponentInfoAdapterException { + public void testGetComponentInfoWithCurationsAndExclusions() + throws ComponentInfoAdapterException, CurationInvalidException { // given this.singleFileCurationProvider @@ -104,9 +108,11 @@ public void testGetComponentInfoWithCurationsAndExclusions() throws ComponentInf * paths are excluded * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException if the curation data is not valid */ @Test - public void testGetComponentInfoWithCurationsAndWithoutExclusions() throws ComponentInfoAdapterException { + public void testGetComponentInfoWithCurationsAndWithoutExclusions() + throws ComponentInfoAdapterException, CurationInvalidException { // given this.singleFileCurationProvider.setCurationsFileName("src/test/resources/scancodefileadapter/curations.yaml"); diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/RawCurationTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/RawCurationTest.java new file mode 100644 index 00000000..d749c27d --- /dev/null +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/RawCurationTest.java @@ -0,0 +1,663 @@ +package com.devonfw.tools.solicitor.componentinfo.scancode; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.devonfw.tools.solicitor.common.packageurl.AllKindsPackageURLHandler; +import com.devonfw.tools.solicitor.componentinfo.ComponentInfo; +import com.devonfw.tools.solicitor.componentinfo.ComponentInfoAdapterException; +import com.devonfw.tools.solicitor.componentinfo.DataStatusValue; +import com.devonfw.tools.solicitor.componentinfo.LicenseInfo; +import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; +import com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider; + +/** + * This class contains JUnit test methods for the testing the raw curations of + * {@link FilteredScancodeComponentInfoProvider} class. + */ +public class RawCurationTest { + + // the object under test + FilteredScancodeComponentInfoProvider filteredScancodeComponentInfoProvider; + + FileScancodeRawComponentInfoProvider fileScancodeRawComponentInfoProvider; + + SingleFileCurationProvider singleFileCurationProvider; + + @BeforeEach + public void setup() { + + AllKindsPackageURLHandler packageURLHandler = Mockito.mock(AllKindsPackageURLHandler.class); + + Mockito.when(packageURLHandler.pathFor("pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0")) + .thenReturn("pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0"); + + this.fileScancodeRawComponentInfoProvider = new FileScancodeRawComponentInfoProvider(packageURLHandler); + this.fileScancodeRawComponentInfoProvider.setRepoBasePath("src/test/resources/scancodefileadapter/Source/repo"); + + this.singleFileCurationProvider = new SingleFileCurationProvider(packageURLHandler); + this.singleFileCurationProvider.setCurationsFileName("src/test/resources/scancodefileadapter/curations.yaml"); + + this.filteredScancodeComponentInfoProvider = new FilteredScancodeComponentInfoProvider( + this.fileScancodeRawComponentInfoProvider, packageURLHandler, this.singleFileCurationProvider); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_AllConditionsSetAndMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_1.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(1, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + assertEquals("Apache-2.0", + scancodeComponentInfo.getComponentInfoData().getLicenses().iterator().next().getSpdxid()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_AllConditionsSetAndPathNotMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_2.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_AllConditionsSetAndRuleIdentifierNotMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_3.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_AllConditionsSetAndOldLicenseNotMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_4.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_AllConditionsSetAndMatchedTextNotMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_5.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_OnlyPathConditionSetAndMetForAllFiles() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_7.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(0, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseReplace_AllConditionsSetAndMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_1.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + boolean found = false; + for (LicenseInfo license : scancodeComponentInfo.getComponentInfoData().getLicenses()) { + if (license.getSpdxid().equals("NewLicense")) { + found = true; + assertEquals("pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3", license.getLicenseUrl()); + assertTrue(license.getGivenLicenseText() + .startsWith(" * This file is part of the test data for deep license scan support in Solicitor..")); + } + } + assertTrue(found); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseReplace_AllConditionsSetAndMetOnlyLicenseReplaced() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_2.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + boolean found = false; + for (LicenseInfo license : scancodeComponentInfo.getComponentInfoData().getLicenses()) { + if (license.getSpdxid().equals("NewLicense")) { + found = true; + assertEquals("pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L4", license.getLicenseUrl()); + assertTrue(license.getGivenLicenseText() + .startsWith(" * It is licensed under the same license as the rest of Solicitor.")); + } + } + assertTrue(found); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseReplace_AllConditionsSetAndMetOnlyUrlReplaced() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_3.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + boolean found = false; + for (LicenseInfo license : scancodeComponentInfo.getComponentInfoData().getLicenses()) { + if (license.getSpdxid().equals("LicenseRef-scancode-unknown-license-reference")) { + found = true; + assertEquals("pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3", license.getLicenseUrl()); + assertTrue(license.getGivenLicenseText() + .startsWith(" * This file is part of the test data for deep license scan support in Solicitor..")); + } + } + assertTrue(found); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseAdd_WithPath() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_add_curation_1.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(3, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + boolean found = false; + for (LicenseInfo license : scancodeComponentInfo.getComponentInfoData().getLicenses()) { + if (license.getSpdxid().equals("NewLicense")) { + found = true; + assertEquals("pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3", license.getLicenseUrl()); + assertTrue(license.getGivenLicenseText() + .startsWith(" * This file is part of the test data for deep license scan support in Solicitor..")); + } + } + assertTrue(found); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseAdd_WithoutPath() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_add_curation_2.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(3, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + boolean found = false; + for (LicenseInfo license : scancodeComponentInfo.getComponentInfoData().getLicenses()) { + if (license.getSpdxid().equals("NewLicense")) { + found = true; + assertEquals("pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3", license.getLicenseUrl()); + assertTrue(license.getGivenLicenseText() + .startsWith(" * This file is part of the test data for deep license scan support in Solicitor..")); + } + } + assertTrue(found); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseAdd_WithPathNotMatching() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/license_add_curation_3.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getLicenses().size()); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightRemove_AllConditionsSetAndMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_1.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(0, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightRemove_AllConditionsSetAndPathNotMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_2.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(1, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + assertEquals("Copyright 2023 devonfw", + scancodeComponentInfo.getComponentInfoData().getCopyrights().iterator().next()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawLicenseRemove_AllConditionsSetAndOldCopyrightNotMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_4.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(1, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + assertEquals("Copyright 2023 devonfw", + scancodeComponentInfo.getComponentInfoData().getCopyrights().iterator().next()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightRemove_OnlyPathConditionSetAndMetForAllFiles() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_7.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(0, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightReplace_AllConditionsSetAndMet() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_replace_curation_1.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(1, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + assertEquals("(c) 2023 devonfw", scancodeComponentInfo.getComponentInfoData().getCopyrights().iterator().next()); + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightAdd_WithPath() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_1.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + assertTrue(scancodeComponentInfo.getComponentInfoData().getCopyrights().contains("(c) 2024 devonfw")); + assertTrue(scancodeComponentInfo.getComponentInfoData().getCopyrights().contains("Copyright 2023 devonfw")); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightAdd_WithoutPath() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_2.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(2, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + assertTrue(scancodeComponentInfo.getComponentInfoData().getCopyrights().contains("(c) 2024 devonfw")); + assertTrue(scancodeComponentInfo.getComponentInfoData().getCopyrights().contains("Copyright 2023 devonfw")); + + } + + /** + * Test the + * {@link FilteredScancodeComponentInfoProvider#getComponentInfo(String, com.devonfw.tools.solicitor.componentinfo.CurationDataHandle)} + * + * + * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException + */ + @Test + public void testGetComponentInfoRawCopyrightAdd_WithPathNotMatching() + throws ComponentInfoAdapterException, CurationInvalidException { + + // given + this.singleFileCurationProvider + .setCurationsFileName("src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_3.yaml"); + + // when + ComponentInfo scancodeComponentInfo = this.filteredScancodeComponentInfoProvider.getComponentInfo( + "pkg:maven/com.devonfw.tools/test-project-for-deep-license-scan@0.1.0", + new SelectorCurationDataHandle("someCurationSelector")); + + // then + assertNotNull(scancodeComponentInfo.getComponentInfoData()); + assertNotEquals(DataStatusValue.CURATED, scancodeComponentInfo.getDataStatus()); + assertEquals(1, scancodeComponentInfo.getComponentInfoData().getCopyrights().size()); + assertTrue(scancodeComponentInfo.getComponentInfoData().getCopyrights().contains("Copyright 2023 devonfw")); + + } + +} diff --git a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfoAdapterTest.java b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfoAdapterTest.java index ad713cba..acb981e5 100644 --- a/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfoAdapterTest.java +++ b/core/src/test/java/com/devonfw/tools/solicitor/componentinfo/scancode/ScancodeComponentInfoAdapterTest.java @@ -20,6 +20,7 @@ import com.devonfw.tools.solicitor.componentinfo.SelectorCurationDataHandle; import com.devonfw.tools.solicitor.componentinfo.curation.ComponentInfoCurator; import com.devonfw.tools.solicitor.componentinfo.curation.ComponentInfoCuratorImpl; +import com.devonfw.tools.solicitor.componentinfo.curation.CurationInvalidException; import com.devonfw.tools.solicitor.componentinfo.curation.CurationProvider; import com.devonfw.tools.solicitor.componentinfo.curation.SingleFileCurationProvider; @@ -75,9 +76,10 @@ public void setup() { * known. * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException */ @Test - public void testGetComponentInfoNaPackage() throws ComponentInfoAdapterException { + public void testGetComponentInfoNaPackage() throws ComponentInfoAdapterException, CurationInvalidException { // given @@ -94,9 +96,10 @@ public void testGetComponentInfoNaPackage() throws ComponentInfoAdapterException * available. * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException */ @Test - public void testGetComponentInfoWithoutCurations() throws ComponentInfoAdapterException { + public void testGetComponentInfoWithoutCurations() throws ComponentInfoAdapterException, CurationInvalidException { // given this.singleFileCurationProvider.setCurationsFileName("src/test/resources/scancodefileadapter/nonexisting.yaml"); @@ -144,9 +147,11 @@ public void testGetComponentInfoWithoutCurations() throws ComponentInfoAdapterEx * curationDataSelector to downstream beans. * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException */ @Test - public void testGetComponentCheckCurationDataSelector() throws ComponentInfoAdapterException { + public void testGetComponentCheckCurationDataSelector() + throws ComponentInfoAdapterException, CurationInvalidException { // given CurationProvider curationProvider = Mockito.mock(CurationProvider.class); @@ -178,9 +183,10 @@ public void testGetComponentCheckCurationDataSelector() throws ComponentInfoAdap * Test the {@link ScancodeComponentInfoAdapter#getComponentInfo(String,String)} method when curations are existing. * * @throws ComponentInfoAdapterException if something goes wrong + * @throws CurationInvalidException */ @Test - public void testGetComponentInfoWithCurations() throws ComponentInfoAdapterException { + public void testGetComponentInfoWithCurations() throws ComponentInfoAdapterException, CurationInvalidException { // when ComponentInfo componentInfo = this.scancodeComponentInfoAdapter.getComponentInfo( diff --git a/core/src/test/resources/curations/array_of_curations.yaml b/core/src/test/resources/curations/array_of_curations.yaml index 5ff9e40a..8adae30c 100644 --- a/core/src/test/resources/curations/array_of_curations.yaml +++ b/core/src/test/resources/curations/array_of_curations.yaml @@ -11,3 +11,17 @@ artifacts: - license: "BSD-2-Clause" url: "https://scancode-licensedb.aboutcode.org/bsd-simplified.LICENSE" copyrights: [] +- name: "pkg/maven/othernamespace" + licenseCurations: + - path: some/path/1 + operation: REMOVE + ruleIdentifier: rule_1 + matchedText: some text 1 + comment: just for testing +- name: "pkg/maven/othernamespace/othercomponent" + licenseCurations: + - path: "some/path/2" + operation: REMOVE + ruleIdentifier: "rule_2" + matchedText: "some text 2" + comment: "just for second test" diff --git a/core/src/test/resources/curations/curation_with_all_fields_set.yaml b/core/src/test/resources/curations/curation_with_all_fields_set.yaml index f1a6cd6d..087f6e15 100644 --- a/core/src/test/resources/curations/curation_with_all_fields_set.yaml +++ b/core/src/test/resources/curations/curation_with_all_fields_set.yaml @@ -11,3 +11,41 @@ licenses: copyrights: - "Copyright (c) 2003 First Holder" - "Copyright (c) 2004 Second Holder" +licenseCurations: +- path: some/path/1 + operation: REMOVE + ruleIdentifier: rule_1 + matchedText: some text 1 + comment: just for testing +- path: "some/path/2" + operation: REMOVE + ruleIdentifier: "rule_2" + matchedText: "some text 2" + comment: "just for second test" +- path: another/path/1 + operation: ADD + newLicense: Apache-2.0 + url: https://scancode-licensedb.aboutcode.org/apache-2.0.LICENSE + comment: just for testing 3 +- path: another/path/2 + operation: ADD + newLicense: MIT + url: https://scancode-licensedb.aboutcode.org/MIT.LICENSE + comment: just for testing 4 +copyrightCurations: +- path: other/path/1 + operation: REMOVE + oldCopyright: "(c) 2024 me" + comment: testing 1 +- path: other/path/2 + operation: REMOVE + oldCopyright: "(c) 2024 you" + comment: testing 2 +- path: some/path/1 + operation: ADD + newCopyright: "(c) 2024 he" + comment: testing 3 +- path: some/path/2 + operation: ADD + newCopyright: "(c) 2024 them" + comment: testing 2 diff --git a/core/src/test/resources/curations/curations.txt b/core/src/test/resources/curations/curations.txt new file mode 100644 index 00000000..8e7bb991 --- /dev/null +++ b/core/src/test/resources/curations/curations.txt @@ -0,0 +1,12 @@ +name: "pkg/maven/ch/qos/logback/logback-classic/1.2.3" +note: "some note" +copyrights: +- "Copyright (c) 1999-2010, QOS.ch" +- "Copyright (c) 1999-2012, QOS.ch" +- "Copyright (c) 1999-2015, QOS.ch" +- "Copyright (c) 1999-2016, QOS.ch" +licenses: +- license: "EPL-1.0" + url: "pkgcontent:/ch/qos/logback/classic/AsyncAppender.java#L5-L12" +- license: "LGPL-2.1-only" + url: "pkgcontent:/ch/qos/logback/classic/AsyncAppender.java#L5-L12" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_1.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_1.yaml new file mode 100644 index 00000000..9d34f7ef --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_1.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: ADD + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + newCopyright: "(c) 2024 devonfw" + comment: "add rule with path" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_2.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_2.yaml new file mode 100644 index 00000000..31b22375 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_2.yaml @@ -0,0 +1,7 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: ADD + newCopyright: "(c) 2024 devonfw" + comment: "add rule with no condition" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_3.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_3.yaml new file mode 100644 index 00000000..6451af7d --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_add_curation_3.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: ADD + path: "sources/src/main/java/com/devonfw/WRONG/tools/test/SampleClass2.java" + newCopyright: "(c) 2024 devonfw" + comment: "add rule with path which does not match" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_1.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_1.yaml new file mode 100644 index 00000000..445832e7 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_1.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass1.java" + oldCopyright: "Copyright 2023 devonfw" + comment: "remove rule with all conditions set" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_2.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_2.yaml new file mode 100644 index 00000000..275d8b3b --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_2.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/WRONG/tools/test/SampleClass1.java" + oldCopyright: "Copyright 2023 devonfw" + comment: "remove rule with all conditions set" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_4.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_4.yaml new file mode 100644 index 00000000..0a021d1d --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_4.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/WRONG/tools/test/SampleClass1.java" + oldCopyright: "Copyright WRONG 2023 devonfw" + comment: "remove rule with all conditions set" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_7.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_7.yaml new file mode 100644 index 00000000..e774b3ff --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_remove_curation_7.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: REMOVE + path: "sources/.*" + # oldCopyright: "Copyright 2023 devonfw" + comment: "remove rule matching to all files/copyrights" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/copyright_replace_curation_1.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_replace_curation_1.yaml new file mode 100644 index 00000000..c8c8a3d6 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/copyright_replace_curation_1.yaml @@ -0,0 +1,9 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + copyrightCurations: + - operation: REPLACE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass1.java" + oldCopyright: "Copyright 2023 devonfw" + newCopyright: "(c) 2023 devonfw" + comment: "replace rule with all conditions set" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_1.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_1.yaml new file mode 100644 index 00000000..23467b44 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_1.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: ADD + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + newLicense: "NewLicense" + url: "pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_2.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_2.yaml new file mode 100644 index 00000000..a8f27c5b --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_2.yaml @@ -0,0 +1,7 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: ADD + newLicense: "NewLicense" + url: "pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_3.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_3.yaml new file mode 100644 index 00000000..c81f3342 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_add_curation_3.yaml @@ -0,0 +1,8 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: ADD + path: "sources/src/main/java/com/devonfw/WRONG/tools/test/SampleClass2.java" + newLicense: "NewLicense" + url: "pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_1.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_1.yaml new file mode 100644 index 00000000..086da595 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_1.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "remove rule with all conditions set" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_2.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_2.yaml new file mode 100644 index 00000000..d837fb6a --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_2.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/src/main/java/WRONG/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "remove rule with all conditions set" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_3.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_3.yaml new file mode 100644 index 00000000..f8608fd7 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_3.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-WRONG-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "remove rule with all conditions set" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_4.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_4.yaml new file mode 100644 index 00000000..ca2d1367 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_4.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-WRONG-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "remove rule with all conditions set" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_5.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_5.yaml new file mode 100644 index 00000000..a08d5759 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_5.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the WRONG same license as the rest of Solicitor.*" + comment: "remove rule with all conditions set" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_6.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_6.yaml new file mode 100644 index 00000000..436f964a --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_6.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + #ruleIdentifier: "license-intro_26.RULE" + #oldLicense: "LicenseRef-scancode-unknown-license-reference" + #matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "remove rule with only path condition set" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_7.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_7.yaml new file mode 100644 index 00000000..0b6b75bf --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_remove_curation_7.yaml @@ -0,0 +1,10 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REMOVE + path: "sources/.*" + #ruleIdentifier: "license-intro_26.RULE" + #oldLicense: "LicenseRef-scancode-unknown-license-reference" + #matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "remove rule with only path condition set which matches for all files" \ No newline at end of file diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_1.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_1.yaml new file mode 100644 index 00000000..36ec5f37 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_1.yaml @@ -0,0 +1,12 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REPLACE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "replace rule with all conditions set" + newLicense: "NewLicense" + url: "pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_2.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_2.yaml new file mode 100644 index 00000000..df5c4ec1 --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_2.yaml @@ -0,0 +1,12 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REPLACE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "replace rule with all conditions set" + newLicense: "NewLicense" + # url: "new url" diff --git a/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_3.yaml b/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_3.yaml new file mode 100644 index 00000000..3c0cd10f --- /dev/null +++ b/core/src/test/resources/scancodefileadapter/rawcurations/license_replace_curation_3.yaml @@ -0,0 +1,12 @@ +artifacts: +- name: "pkg/maven/com/devonfw/tools/test-project-for-deep-license-scan/0.1.0" + note: "some note on the curation" + licenseCurations: + - operation: REPLACE + path: "sources/src/main/java/com/devonfw/tools/test/SampleClass2.java" + ruleIdentifier: "license-intro_26.RULE" + oldLicense: "LicenseRef-scancode-unknown-license-reference" + matchedText: ".*It is licensed under the same license as the rest of Solicitor.*" + comment: "replace rule with all conditions set" + #newLicense: "NewLicense" + url: "pkgcontent:/src/main/java/com/devonfw/tools/test/SampleClass2.java#L3" diff --git a/documentation/master-solicitor.asciidoc b/documentation/master-solicitor.asciidoc index 78aaae4a..4beffda3 100644 --- a/documentation/master-solicitor.asciidoc +++ b/documentation/master-solicitor.asciidoc @@ -1561,11 +1561,125 @@ solicitor.feature-flag.scancode.automapping=false === Correcting data The data obtained from ScanCode might be affected by false positives (wrongly detected a license or copyright) or false negatives (missed to detect a license or copyright). To compensate such defects there are two mechanisms: -Applying Curation information from a "curations" file or changing the License information via the decision table rules. +Applying Curation information from a "curations" file or changing the license information via the decision table rules. -==== Curations file +==== Curating data via a curations file + +===== Structure of curations file To define curations you might create a file `output/curations.yaml` containing the following structure: +[source,yaml] +---- +artifacts: + - name: pkg/npm/@somescope/somepackage/1.2.3 <1> + url: https://github.com/foo/bar <2> + licenseCurations: <3> + - operation: REMOVE + path: "sources/package/readme.md" + ruleIdentifier: "proprietary-license_unknown_13.RULE" + matchedText: ".* to be paid .*" + comment: "just a generic remark, not a license" + - operation: ADD + newLicense: "Apache-2.0" + comment: "License as given on website" + copyrightCurations: <4> + - operation: REMOVE + path: "sources/package/lib/test.js" + oldCopyright: "(c) R.apv" + comment: "some minified code fragment, not a copyright" + excludedPaths: <5> + - "sources/src" <6> + - name: pkg/npm/@anotherscope/anotherpackage/4.5.6 <7> +. +. +. +---- +<1> Path of the package information as used in the file tree. Derived from the PackageURL. +<2> URL of the project, will be stored as `sourceRepoUrl`. (Optional: no change if not existing.) +<3> Rules for curating license findings, see below. +<4> Rules for curation copyright findings, see below. +<5> Excluded paths to be set. Optional. If defined then all scanned files, whose path prefix contain any given string here, are excluded from the ScanCode information. +<6> A single path prefix. All scanned files starting with this path prefix are excluded from the Scancode information. +<7> Further packages to follow. + +===== Rules for curating licenses +Curating licenses is done by REMOVING (i.e. ignoring) specifc license findings from ScanCode, by REPLACING the detected license with another one or by ADDING license findings either to specific files or on top level (not related to specific file of the package sources). +In addition to the conditions/data which is specific for any of the below described operations it is always possible to define a comment which is intended to be included in any audit trail log for documentation porposes (not yet used/implemented). + + +====== Licenses: REMOVE +Removing found licenses is done by defining rules which result in ignoring the license finding(s) of scancode rules in files within the scanned codebase. The following "conditions" are used for defining the rule + +* `path` of the file within the sources (defined as a regular expression; matches to `files[].path` in the scancode json file) +* `ruleIdentifier` of the rule (defined as a regular expression; matches to `files[].licenses[].matched_rule.identifier` in the scancode json file) +* `matchedText` of the finding (defined as a regular expression; matches to `files[].licenses[].matched_text` in the scancode json file) +* `oldLicense` of the finding (defined as regular expression; matches to `files[].licenses[].spdx_license_key` + +The first three conditions can uniquely identify any license finding listed in the scancode json file. The `oldLicense` condition can be used to select findings to be ignored based on the found license instead of the `ruleIdentifier`. +All conditions are optional but at least one needs to be defined. By using RegEx syntax the curations can be written very flexible. By using solely `oldLicense` as a condition it is e.g. possible to remove all findings of a specific license. + +====== Licenses: REPLACE +Instead of removing licenses (ignoring the finding) they might be replaced with a different license key and/or URL pointing to the license text. The conditions are the same as for REMOVE, the replacement is defined as follows + +Data: + +* `newLicense` is the key / id of the license to use instead (replacing `files[].licenses[].spdx_license_key`) +* `url` is the url pointing to the license text + +At least one of the two parameters has to be set. + +====== Licenses: ADD +Adding new licenses is done by defining rules which add new license info (to the licenses found in a source file) - or "on top level". + +Conditions: + +* `path` of the file within the sources to which the license should be added (defined as a regular expression; matches to `files[].path` in the scancode json file). Note that this will only work if there are `files[].path` in the scancode json for which this conditions matches. + +It is not possible to associate licenses to files which are not listed in scancode json. The `path` condition might be omitted which results in the given license to be added to the result without any relation to a specific path. + +Data: + +* `newLicense`: the key/spdxid of the license to add +* `url`: URL to the license text + +===== Rules for curating copyrights +Curating copyrights is based on the same principles as curation of licenses, providing REMOVE, REPLACE and ADD operations. + + +====== Copyrights: REMOVE +REMOVING found copyrights is done by defining rules which result in ignoring the copyright finding(s) in files within the scanned codebase. The following "conditions" are used for defining the rule + +* `path` of the file within the sources (defined as a regular expression; matches to `files[].path` in the scancode json file) +* `oldCopyright` the found copyright text to ignore (defined as a regular expression; matches to `files[].copyrights[].copyright` in the scancode json file) + +At least one of the conditions has to be defined. + +====== Copyright: REPLACE +This follows the above principles. It uses the same conditions as REMOVE and uses a parameter to define the copyright to use instead: + +Data: +* `newCopyright`: The copyright entry to use instead of the originally found copyright + +====== Copyright: ADD +Adding new copyrights is done by defining rules which add new copyright info (to the copyrights found in a source file) - or "on top level". + +Conditions: + +* `path` of the file within the sources (defined as a regular expression; if omitted the copyright will be applied on "top level"). +Note that it is again only possible to add copyrigts to paths which are listed in the scancode json + +Data: + +* `newCopyright`: the copyright string to add + +===== Redefining all licenses / copyrights of a component + +Instead of curating license / copyrights on a "per finding" level as given above it is alternatively possible to completely replace the list of found licenses and/or copyrights with a new list. + +IMPORTANT: Up to version 1.23.0 this was the only way of doing license / copyright curations. Use of this way of curating data is still possible but discouraged and might be deprecated/removed soon. + +The file `output/curations.yaml` looks as follows when doing curations this way: + [source,yaml] ---- artifacts: @@ -1596,6 +1710,33 @@ artifacts: <10> A single path prefix. All scanned files starting with this path prefix are excluded from the Scancode information. <11> Further packages to follow. +===== Hierarchical definition of rules + +Different version of a package/component or even different packages/components within the same namespace often require mostly the same curations to be applied. To avoid being forced to redefine curations for every single version it is possible to define curations by just specifying a prefix part in the name attribute. + +Example of available levels/prefixes for `pkg:/maven/ch.qos.logback/logback-classic@1.2.3` + +* `pkg` +* `pkg/maven` +* `pkg/maven/ch` +* `pkg/maven/ch/qos` +* `pkg/maven/ch/qos/logback` +* `pkg/maven/ch/qos/logback/logback-classic` +* `pkg/maven/ch/qos/logback/logback-classic/1.2.3` + +The complete tree will be checked for curations. Any found curations will be merged + +* Attribute `name`: latest encountered in the hierarchy will be taken +* Attribute `note` will be joined using delimiter " / " +* Attribute `url`: latest encountered in the hierarchy will be taken +* Attribute `copyrights` (old style of curations): Lists will be merged +* Attribute `licenses` (old style of curations): Lists will be merged +* Attribute `excludedPaths`: Lists will be merged +* Attribute `licenseCurations`: License curation rule lists (REMOVE/REPLACE/ADD) will be merged; order is more specific ones first; when evaluating for a specific license finding in the scancode json only the first matching curation rule will be taken. +* Attribute `copyrightCurations`: Copyright curation rule lists (REMOVE/REPLACE/ADD) will be merged; order is more specific ones first; when evaluating for a specific copyright finding in the scancode json only the first matching curation rule will be taken. + +The resulting curation will then be applied to the scancode data of the component. + ==== Decision table rules As for license information obtained from the Readers the license information from ScanCode can also be altered using decision table rules. A new attribute `origin` was introduced in the `RawLicense` entity as well as condition field in decision table `LicenseAssignmentV2*.xls/csv`. The `origin` attribute in `Rawlicense` either contains the string `scancode` if the license information came from ScanCode or it contains the (lowercase) class name of the used Reader. @@ -1758,6 +1899,7 @@ Changes in 1.24.0:: * https://github.com/devonfw/solicitor/issues/263: Some features were pushed from deprecation stage 1 to stage 2, which means they do not longer work with default configuration: repoType attribute, npm and npm-license-crawler readers, REGEX prefix notation, LicenseAssignmentProject.xls. See <> for details. * https://github.com/devonfw/solicitor/pull/265: Added some license name mappings. * https://github.com/devonfw/solicitor/pull/266: Improve correlation of records (diff processing, see <>) for aggregated inventory report `OSS-Inventory_aggregated_*.xlsx`. +* https://github.com/devonfw/solicitor/issues/267: Introduce more fine grained and hierarchical curations of licenses and copyrights. See <>. Changes in 1.23.0:: * https://github.com/devonfw/solicitor/issues/255: Deprecate LicenseUrl guessing.