Skip to content
This repository has been archived by the owner on Jul 25, 2018. It is now read-only.

Commit

Permalink
feat(licenseinfo): make licenseinfo generation more robust
Browse files Browse the repository at this point in the history 
display error messages at license selection screen and in the generated
license info files when one of the attachments cannot be parsed, as opposed
to failing outright.

merge license info data when multiple CLI attachments were selected by
the user (only for text and xhtml generators)

closes #606
  • Loading branch information
@alexbrdn
alexbrdn committed Feb 1, 2018
1 parent 2a5162e commit 9a36683
Show file tree
Hide file tree
Showing 8 changed files with 187 additions and 83 deletions.
50 changes: 32 additions & 18 deletions ...d/src/src-licenseinfo/src/main/java/org/eclipse/sw360/licenseinfo/LicenseInfoHandler.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright Siemens AG, 2016-2017. Part of the SW360 Portal Project.
* Copyright Siemens AG, 2016-2018. Part of the SW360 Portal Project.
*
* SPDX-License-Identifier: EPL-1.0
*
Expand All @@ -16,7 +16,6 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.eclipse.sw360.attachments.db.AttachmentDatabaseHandler;
Expand All @@ -36,13 +35,13 @@
import org.eclipse.sw360.licenseinfo.parsers.*;
import org.eclipse.sw360.licenseinfo.util.LicenseNameWithTextUtils;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.eclipse.sw360.datahandler.common.CommonUtils.nullToEmptySet;
import static org.eclipse.sw360.datahandler.common.SW360Assert.assertNotNull;
import static org.eclipse.sw360.datahandler.common.WrappedException.wrapTException;

Expand Down Expand Up @@ -136,15 +135,15 @@ public OutputFormatInfo getOutputFormatInfoForGeneratorClass(String generatorCla
public List<LicenseInfoParsingResult> getLicenseInfoForAttachment(Release release, String attachmentContentId, User user)
throws TException {
if (release == null) {
return assignReleaseToLicenseInfoParsingResult(noSourceParsingResult(), release);
return Collections.singletonList(noSourceParsingResult("No release given"));
}

List<LicenseInfoParsingResult> cachedResults = licenseInfoCache.getIfPresent(attachmentContentId);
if (cachedResults != null) {
return cachedResults;
}

Attachment attachment = CommonUtils.nullToEmptySet(release.getAttachments()).stream()
Attachment attachment = nullToEmptySet(release.getAttachments()).stream()
.filter(a -> a.getAttachmentContentId().equals(attachmentContentId)).findFirst().orElseThrow(() -> {
String message = String.format(
"Attachment selected for license info generation is not found in release's attachments. Release id: %s. Attachment content id: %s",
Expand All @@ -159,7 +158,10 @@ public List<LicenseInfoParsingResult> getLicenseInfoForAttachment(Release releas

if (applicableParsers.size() == 0) {
LOGGER.warn("No applicable parser has been found for the attachment selected for license information");
return assignReleaseToLicenseInfoParsingResult(noSourceParsingResult(), release);
return assignReleaseToLicenseInfoParsingResult(
assignFileNameToLicenseInfoParsingResult(
noSourceParsingResult("No applicable parser has been found for the attachment"), attachment.getFilename()),
release);
} else if (applicableParsers.size() > 1) {
LOGGER.info("More than one parser claims to be able to parse attachment with contend id " + attachmentContentId);
}
Expand All @@ -177,6 +179,14 @@ public List<LicenseInfoParsingResult> getLicenseInfoForAttachment(Release releas
}
}

private LicenseInfoParsingResult assignFileNameToLicenseInfoParsingResult(LicenseInfoParsingResult licenseInfoParsingResult, String filename) {
if (licenseInfoParsingResult.getLicenseInfo() == null) {
licenseInfoParsingResult.setLicenseInfo(new LicenseInfo());
}
licenseInfoParsingResult.getLicenseInfo().addToFilenames(filename);
return licenseInfoParsingResult;
}

@Override
public String getDefaultLicenseInfoHeaderText() {
return DEFAULT_LICENSE_INFO_TEXT;
Expand Down Expand Up @@ -225,23 +235,27 @@ protected Collection<LicenseInfoParsingResult> getAllReleaseLicenseInfos(Map<Rel
}

protected LicenseInfoParsingResult filterLicenses(LicenseInfoParsingResult result, Set<LicenseNameWithText> licencesToExclude) {
Set<LicenseNameWithText> filteredLicenses = result.getLicenseInfo().getLicenseNamesWithTexts().stream().filter(license -> {
for (LicenseNameWithText excludeLicense : licencesToExclude) {
if (LicenseNameWithTextUtils.licenseNameWithTextEquals(license, excludeLicense)) {
return false;
}
}
return true;
}).collect(Collectors.toSet());

// make a deep copy to NOT change the original document that is cached
LicenseInfoParsingResult newResult = result.deepCopy();
newResult.getLicenseInfo().setLicenseNamesWithTexts(filteredLicenses);

if (result.getLicenseInfo() != null) {
Set<LicenseNameWithText> filteredLicenses = nullToEmptySet(result.getLicenseInfo().getLicenseNamesWithTexts())
.stream()
.filter(license -> {
for (LicenseNameWithText excludeLicense : licencesToExclude) {
if (LicenseNameWithTextUtils.licenseNameWithTextEquals(license, excludeLicense)) {
return false;
}
}
return true;
}).collect(Collectors.toSet());
newResult.getLicenseInfo().setLicenseNamesWithTexts(filteredLicenses);
}
return newResult;
}

protected LicenseInfoParsingResult noSourceParsingResult() {
return new LicenseInfoParsingResult().setStatus(LicenseInfoRequestStatus.NO_APPLICABLE_SOURCE);
protected LicenseInfoParsingResult noSourceParsingResult(String message) {
return new LicenseInfoParsingResult().setStatus(LicenseInfoRequestStatus.NO_APPLICABLE_SOURCE).setMessage(message);
}

protected List<LicenseInfoParsingResult> assignReleaseToLicenseInfoParsingResult(LicenseInfoParsingResult licenseInfoParsingResult,
Expand Down
80 changes: 61 additions & 19 deletions ...c-licenseinfo/src/main/java/org/eclipse/sw360/licenseinfo/outputGenerators/DocxUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/*
* Copyright Bosch Software Innovations GmbH, 2016.
* With modifications by Siemens AG, 2018.
* Part of the SW360 Portal Project.
*
* SPDX-License-Identifier: EPL-1.0
Expand All @@ -15,6 +16,7 @@
import org.apache.poi.xwpf.usermodel.*;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfo;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfoParsingResult;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfoRequestStatus;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseNameWithText;

import java.util.*;
Expand All @@ -25,6 +27,7 @@
public class DocxUtils {

private static final int FONT_SIZE = 12;
public static final String ALERT_COLOR = "e95850";

private DocxUtils() {
//only static members
Expand Down Expand Up @@ -73,29 +76,37 @@ private static void styleTable(XWPFTable table) {
public static void fillReleasesTable(XWPFTable table, Collection<LicenseInfoParsingResult> projectLicenseInfoResults) {

for (LicenseInfoParsingResult result : projectLicenseInfoResults) {
Set<String> copyrights = Collections.EMPTY_SET;
Set<LicenseNameWithText> licenseNamesWithTexts = Collections.EMPTY_SET;
Set<String> acknowledgements = Collections.EMPTY_SET;
if (result.isSetLicenseInfo()) {
LicenseInfo licenseInfo = result.getLicenseInfo();
if (licenseInfo.isSetCopyrights()) {
copyrights = licenseInfo.getCopyrights();
}
if (licenseInfo.isSetLicenseNamesWithTexts()) {
licenseNamesWithTexts = licenseInfo.getLicenseNamesWithTexts();
acknowledgements = licenseNamesWithTexts.stream()
.map(LicenseNameWithText::getAcknowledgements)
.filter(Objects::nonNull).collect(Collectors.toSet());
}
}
String releaseName = nullToEmptyString(result.getName());
String version = nullToEmptyString(result.getVersion());
if (result.getStatus()== LicenseInfoRequestStatus.SUCCESS) {
Set<String> copyrights = Collections.emptySet();
Set<LicenseNameWithText> licenseNamesWithTexts = Collections.emptySet();
Set<String> acknowledgements = Collections.emptySet();
if (result.isSetLicenseInfo()) {
LicenseInfo licenseInfo = result.getLicenseInfo();
if (licenseInfo.isSetCopyrights()) {
copyrights = licenseInfo.getCopyrights();
}
if (licenseInfo.isSetLicenseNamesWithTexts()) {
licenseNamesWithTexts = licenseInfo.getLicenseNamesWithTexts();
acknowledgements = licenseNamesWithTexts.stream()
.map(LicenseNameWithText::getAcknowledgements)
.filter(Objects::nonNull).collect(Collectors.toSet());
}
}

addTableRow(table, releaseName, version, licenseNamesWithTexts, acknowledgements, copyrights);
addReleaseTableRow(table, releaseName, version, licenseNamesWithTexts, acknowledgements, copyrights);
} else {
String filename = Optional.ofNullable(result.getLicenseInfo())
.map(LicenseInfo::getFilenames)
.map(l -> l.stream().findFirst().orElse(null))
.orElse("");
addReleaseTableErrorRow(table, releaseName, version, nullToEmptyString(result.getMessage()), filename);
}
}
}

private static void addTableRow(XWPFTable table, String releaseName, String version, Set<LicenseNameWithText> licenseNamesWithTexts, Set<String> acknowledgements, Set<String> copyrights) {
private static void addReleaseTableRow(XWPFTable table, String releaseName, String version, Set<LicenseNameWithText> licenseNamesWithTexts, Set<String> acknowledgements, Set<String> copyrights) {
XWPFTableRow row = table.createRow();

XWPFParagraph currentParagraph = row.getCell(0).getParagraphs().get(0);
Expand Down Expand Up @@ -135,6 +146,30 @@ private static void addTableRow(XWPFTable table, String releaseName, String vers
}
}

private static void addReleaseTableErrorRow(XWPFTable table, String releaseName, String version, String error, String filename) {
XWPFTableRow row = table.createRow();

XWPFParagraph currentParagraph = row.getCell(0).getParagraphs().get(0);
styleTableHeaderParagraph(currentParagraph);
XWPFRun currentRun = currentParagraph.createRun();
addFormattedText(currentRun, releaseName, FONT_SIZE);

currentParagraph = row.getCell(1).getParagraphs().get(0);
styleTableHeaderParagraph(currentParagraph);
currentRun = currentParagraph.createRun();
addFormattedText(currentRun, version, FONT_SIZE);

currentParagraph = row.getCell(2).getParagraphs().get(0);
styleTableHeaderParagraph(currentParagraph);
currentRun = currentParagraph.createRun();
addFormattedText(currentRun, String.format("Error reading license information: %s", error), FONT_SIZE, false, ALERT_COLOR);

currentParagraph = row.getCell(4).getParagraphs().get(0);
styleTableHeaderParagraph(currentParagraph);
currentRun = currentParagraph.createRun();
addFormattedText(currentRun, String.format("Source file: %s", filename), FONT_SIZE, false, ALERT_COLOR);
}

private static void styleTableHeaderParagraph(XWPFParagraph paragraph) {
paragraph.setIndentationLeft(0);
paragraph.setWordWrap(true);
Expand Down Expand Up @@ -195,15 +230,22 @@ private static void setText(XWPFRun run, String text) {
}
}

private static void addFormattedText(XWPFRun run, String text, String fontFamily, int fontSize, boolean bold) {
private static void addFormattedText(XWPFRun run, String text, String fontFamily, int fontSize, boolean bold, String rrggbbColor) {
run.setFontSize(fontSize);
run.setFontFamily(fontFamily);
run.setBold(bold);
if (rrggbbColor != null) {
run.setColor(rrggbbColor);
}
setText(run, text);
}

private static void addFormattedText(XWPFRun run, String text, int fontSize, boolean bold, String rrggbbColor) {
addFormattedText(run, text, "Calibri", fontSize, bold, rrggbbColor);
}

private static void addFormattedText(XWPFRun run, String text, int fontSize, boolean bold) {
addFormattedText(run, text, "Calibri", fontSize, bold);
addFormattedText(run, text, "Calibri", fontSize, bold, null);
}

private static void addFormattedText(XWPFRun run, String text, int fontSize) {
Expand Down
46 changes: 33 additions & 13 deletions ...nseinfo/src/main/java/org/eclipse/sw360/licenseinfo/outputGenerators/OutputGenerator.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* Copyright Bosch Software Innovations GmbH, 2016.
* With modifications by Siemens AG, 2017.
* With modifications by Siemens AG, 2017-2018.
* Part of the SW360 Portal Project.
*
* SPDX-License-Identifier: EPL-1.0
Expand All @@ -22,10 +22,7 @@
import org.apache.velocity.tools.ToolManager;
import org.eclipse.sw360.datahandler.common.SW360Utils;
import org.eclipse.sw360.datahandler.thrift.SW360Exception;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfo;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfoParsingResult;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseNameWithText;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.OutputFormatInfo;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.*;
import org.eclipse.sw360.licenseinfo.util.LicenseNameWithTextUtils;
import org.jetbrains.annotations.NotNull;

Expand All @@ -41,6 +38,7 @@ public abstract class OutputGenerator<T> {
protected static final String ALL_LICENSE_NAMES_WITH_TEXTS = "allLicenseNamesWithTexts";
protected static final String LICENSES_CONTEXT_PROPERTY = "licenses";
protected static final String LICENSE_INFO_RESULTS_CONTEXT_PROPERTY = "licenseInfoResults";
protected static final String LICENSE_INFO_ERROR_RESULTS_CONTEXT_PROPERTY = "licenseInfoErrorResults";
protected static final String LICENSE_INFO_HEADER_TEXT = "licenseInfoHeader";

private final String outputType;
Expand Down Expand Up @@ -99,10 +97,23 @@ public VelocityContext getConfiguredVelocityContext() {
@NotNull
protected SortedMap<String, LicenseInfoParsingResult> getSortedLicenseInfos(Collection<LicenseInfoParsingResult> projectLicenseInfoResults) {
Map<String, LicenseInfoParsingResult> licenseInfos = projectLicenseInfoResults.stream()
.collect(Collectors.toMap(this::getComponentLongName, li -> li, (li1, li2) -> li1));
.collect(Collectors.toMap(this::getComponentLongName, li -> li, this::mergeLicenseInfoParsingResults));
return sortStringKeyedMap(licenseInfos);
}

@NotNull
protected LicenseInfoParsingResult mergeLicenseInfoParsingResults(LicenseInfoParsingResult r1, LicenseInfoParsingResult r2){
if (r1.getStatus() != LicenseInfoRequestStatus.SUCCESS || r2.getStatus() != LicenseInfoRequestStatus.SUCCESS ||
!getComponentLongName(r1).equals(getComponentLongName(r2))){
throw new IllegalArgumentException("Only successful parsing results for the same release can be merged");
}
LicenseInfoParsingResult r = new LicenseInfoParsingResult(r1);
r.getLicenseInfo().getLicenseNamesWithTexts().addAll(r2.getLicenseInfo().getLicenseNamesWithTexts());
r.getLicenseInfo().getCopyrights().addAll(r2.getLicenseInfo().getCopyrights());
r.getLicenseInfo().getFilenames().addAll(r2.getLicenseInfo().getFilenames());
return r;
}

@NotNull
protected SortedMap<String, Set<String>> getSortedAcknowledgements(Map<String, LicenseInfoParsingResult> sortedLicenseInfos) {
Map<String, Set<String>> acknowledgements = Maps.filterValues(Maps.transformValues(sortedLicenseInfos, pr -> Optional
Expand Down Expand Up @@ -155,7 +166,10 @@ private static <U> SortedMap<String, U> sortStringKeyedMap(Map<String, U> unsort
* id inside the file. May be used to reference a license.</li>
* <li>licenseInfoResults: map of {@link LicenseInfoParsingResult} objects,
* where the key is the name of the release. The licenses within the objects are
* sorted by name.</li>
* sorted by name. Contains only the results with status {@link LicenseInfoRequestStatus#SUCCESS}</li>
* <li>licenseInfoErrorResults: map {@link List}of {@link LicenseInfoParsingResult} objects,
* where the key is the name of the release. Contains only the results with status other than
* {@link LicenseInfoRequestStatus#SUCCESS}. These results are not merged, that's why the map values are lists.</li>
* <li>acknowledgments: map of acknowledgments for a release where the key is
* the release and the value is a set of strings (acknowledgments)</li>
* </ul>
Expand All @@ -175,7 +189,7 @@ protected String renderTemplateWithDefaultValues(Collection<LicenseInfoParsingRe

// sorted lists of all license to be displayed at the end of the file at once
List<LicenseNameWithText> licenseNamesWithTexts = getSortedLicenseNameWithTexts(projectLicenseInfoResults);
vc.put(OutputGenerator.ALL_LICENSE_NAMES_WITH_TEXTS, licenseNamesWithTexts);
vc.put(ALL_LICENSE_NAMES_WITH_TEXTS, licenseNamesWithTexts);

// assign a reference id to each license in order to only display references for
// each release. The references will point to
Expand All @@ -187,17 +201,23 @@ protected String renderTemplateWithDefaultValues(Collection<LicenseInfoParsingRe
}
vc.put(LICENSE_REFERENCE_ID_MAP_CONTEXT_PROPERTY, licenseToReferenceId);

Map<Boolean, List<LicenseInfoParsingResult>> partitionedResults =
projectLicenseInfoResults.stream().collect(Collectors.partitioningBy(r -> r.getStatus() == LicenseInfoRequestStatus.SUCCESS));
List<LicenseInfoParsingResult> goodResults = partitionedResults.get(true);
Map<String, List<LicenseInfoParsingResult>> badResultsPerRelease =
partitionedResults.get(false).stream().collect(Collectors.groupingBy(this::getComponentLongName));
vc.put(LICENSE_INFO_ERROR_RESULTS_CONTEXT_PROPERTY, badResultsPerRelease);

// be sure that the licenses inside a release are sorted by id. This looks nicer
SortedMap<String, LicenseInfoParsingResult> sortedLicenseInfos = getSortedLicenseInfos(projectLicenseInfoResults);
SortedMap<String, LicenseInfoParsingResult> sortedLicenseInfos = getSortedLicenseInfos(goodResults);
// this will effectively change the objects in the collection and therefore the
// objects in the sorted map above
sortLicenseNamesWithinEachLicenseInfoById(projectLicenseInfoResults, licenseToReferenceId);
vc.put(OutputGenerator.LICENSE_INFO_RESULTS_CONTEXT_PROPERTY, sortedLicenseInfos);
sortLicenseNamesWithinEachLicenseInfoById(sortedLicenseInfos.values(), licenseToReferenceId);
vc.put(LICENSE_INFO_RESULTS_CONTEXT_PROPERTY, sortedLicenseInfos);

// also display acknowledgments
SortedMap<String, Set<String>> acknowledgements = getSortedAcknowledgements(sortedLicenseInfos);
vc.put(OutputGenerator.ACKNOWLEDGEMENTS_CONTEXT_PROPERTY, acknowledgements);

vc.put(ACKNOWLEDGEMENTS_CONTEXT_PROPERTY, acknowledgements);

StringWriter sw = new StringWriter();
Velocity.mergeTemplate(file, "utf-8", vc, sw);
Expand Down
5 changes: 3 additions & 2 deletions backend/src/src-licenseinfo/src/main/resources/textLicenseInfoFile.vm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ $licenseInfoHeader

Open Source Software Contained in this Product/Device:
======================================================
#foreach($releaseName in $licenseInfoResults.keySet())
* $releaseName
#foreach($releaseName in $licenseInfoResults.keySet()) #set($errorResults=[])
* $releaseName #foreach($errorResult in $licenseInfoErrorResults.get($releaseName))
(ERROR when reading license information from file $errorResult.licenseInfo.filenames[0]: $errorResult.message) #end
#end


Expand Down
Loading

0 comments on commit 9a36683

Please sign in to comment.