Skip to content

Commit

Permalink
Tests: Use swap file (not temp dir) for generated tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@petervdonovan
petervdonovan committed Jan 12, 2022
1 parent 909e433 commit 613b9f9
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 97 deletions.
115 changes: 48 additions & 67 deletions org.lflang.tests/src/org/lflang/tests/lsp/ErrorInserter.java
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,24 @@
package org.lflang.tests.lsp;

import java.io.File;
import java.io.Closeable;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Random;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.eclipse.xtext.util.RuntimeIOException;
import org.jetbrains.annotations.NotNull;

import org.lflang.tests.TestRegistry;

import com.google.common.collect.ImmutableList;

/**
Expand All @@ -39,15 +36,16 @@ class ErrorInserter {
.replacer(".get", ".undefined_name15291838")
.replacer("std::", "undefined_name3286634::");
public static final Builder PYTHON = new Builder()
.insertable("+++++").replacer(".", "..");
.insertable("+++++").replacer("print(", "undefined_name15291838(");
public static final Builder RUST = BASE_ERROR_INSERTER
.replacer("println!", "undefined_name15291838!")
.replacer("ctx", "undefined_name3286634!");
.replacer("ctx.", "undefined_name3286634.");
public static final Builder TYPESCRIPT = BASE_ERROR_INSERTER
.replacer("requestErrorStop(", "not_an_attribute_of_util9764(")
.replacer("const ", "var ");

static class AlteredTest {
/** An {@code AlteredTest} represents an altered version of what was a valid LF file. */
static class AlteredTest implements Closeable {

/** A {@code OnceTrue} is randomly true once, and then never again. */
private static class OnceTrue {
Expand All @@ -65,8 +63,8 @@ private boolean get() {

/** The zero-based indices of the touched lines. */
private final List<Integer> badLines;
/** The file to which the altered version of the original LF file should be written. */
private final File file;
/** The original test on which this is based. */
private final Path path;
/** The content of this test. */
private final LinkedList<String> lines;

Expand All @@ -77,27 +75,43 @@ private boolean get() {
*/
public AlteredTest(Path originalTest) throws IOException {
this.badLines = new ArrayList<>();
this.file = tempFileOf(originalTest).toFile();
this.path = originalTest;
this.lines = new LinkedList<>(); // Constant-time insertion during iteration is desired.
this.lines.addAll(Files.readAllLines(originalTest));
}

/** Return the file where the content of {@code this} lives. */
public File getFile() {
return file;
/** Return the location where the content of {@code this} lives. */
public Path getPath() {
return path;
}

/**
* Write the altered version of the test to the file system.
* @throws IOException If an I/O error occurred.
*/
public void write() throws IOException {
if (!file.exists()) copyTestDir();
try (PrintWriter writer = new PrintWriter(file)) {
if (!path.toFile().renameTo(swapFile(path).toFile())) {
throw new IOException("Failed to create a swap file.");
}
try (PrintWriter writer = new PrintWriter(path.toFile())) {
lines.forEach(writer::println);
}
}

/**
* Restore the file associated with this test to its original state.
*/
@Override
public void close() throws IOException {
if (!swapFile(path).toFile().exists()) throw new IllegalStateException("Swap file does not exist.");
if (!path.toFile().delete()) {
throw new IOException("Failed to delete the file associated with the original test.");
}
if (!swapFile(path).toFile().renameTo(path.toFile())) {
throw new IOException("Failed to restore the altered LF file to its original state.");
}
}

/** Return the lines where this differs from the test from which it was derived. */
public ImmutableList<Integer> getBadLines() {
return ImmutableList.copyOf(badLines);
Expand Down Expand Up @@ -158,46 +172,9 @@ private void alter(BiFunction<ListIterator<String>, String, Boolean> alterer) {
}
}

/**
* Return the file location of the temporary copy of {@code test}.
* @param test An LF file that can be used in tests.
* @return The file location of the temporary copy of {@code test}.
*/
private static Path tempFileOf(Path test) {
return ALTERED_TEST_DIR.resolve(TestRegistry.LF_TEST_PATH.relativize(test));
}

/**
* Initialize the error insertion process by recursively copying the test directory to a temporary directory.
* @throws IOException If an I/O error occurs.
*/
private static void copyTestDir() throws IOException {
Files.walkFileTree(TestRegistry.LF_TEST_PATH, new SimpleFileVisitor<>() {

private int depth = 0;

@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
if (depth == 2 && !dir.getName(dir.getNameCount() - 1).toString().equals("src")) {
return FileVisitResult.SKIP_SUBTREE;
}
depth++;
Files.createDirectories(tempFileOf(dir));
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.copy(file, ALTERED_TEST_DIR.resolve(TestRegistry.LF_TEST_PATH.relativize(file)));
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
depth--;
return FileVisitResult.CONTINUE;
}
});
/** Return the swap file associated with {@code f}. */
private static Path swapFile(Path p) {
return p.getParent().resolve("." + p.getFileName() + ".swp");
}
}

Expand Down Expand Up @@ -257,7 +234,20 @@ private Builder(Node<Function<String, String>> replacers, Node<String> insertabl
* replaces {@code phrase} with {@code alternativePhrase}.
*/
public Builder replacer(String phrase, String alternativePhrase) {
return new Builder(new Node<>(replacers, line -> line.replace(phrase, alternativePhrase)), insertables);
return new Builder(
new Node<>(
replacers,
line -> {
int changeableEnd = line.length();
for (String bad : new String[]{"#", "//", "\""}) {
if (line.contains(bad)) changeableEnd = Math.min(changeableEnd, line.indexOf(bad));
}
return line.substring(0, changeableEnd).replace(phrase, alternativePhrase)
+ line.substring(changeableEnd);
}
),
insertables
);
}

/** Record that {@code} line may be inserted in order to introduce an error. */
Expand All @@ -272,15 +262,6 @@ public ErrorInserter get(Random random) {
}

private static final int MAX_ALTERATION_ATTEMPTS = 100;
private static final Path ALTERED_TEST_DIR;

static {
try {
ALTERED_TEST_DIR = Files.createTempDirectory("lingua-franca-altered-tests");
} catch (IOException e) {
throw new RuntimeIOException(e);
}
}

private final Random random;
private final ImmutableList<Function<String, String>> replacers;
Expand Down
78 changes: 49 additions & 29 deletions org.lflang.tests/src/org/lflang/tests/lsp/LspTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class LspTests {
/** The {@code Random} whose initial state determines the behavior of the set of all {@code LspTests} instances. */
private static final Random RANDOM = new Random(2101);
/** The maximum number of integration tests to validate for each target and category. */
private static final int MAX_VALIDATIONS_PER_CATEGORY = 7;
private static final int MAX_VALIDATIONS_PER_CATEGORY = 1000000;
/** The test categories that should be excluded from LSP tests. */
private static final TestCategory[] EXCLUDED_CATEGORIES = {
TestCategory.EXAMPLE, TestCategory.DOCKER, TestCategory.DOCKER_FEDERATED
Expand All @@ -59,32 +59,46 @@ class LspTests {
@Test
void lspWithDependenciesTestRust() { buildAndRunTest(Target.Rust); }

/** Test basic target language validation that should always be present, even without optional dependencies. */
/** Test for false negatives in C++ validation. */
@Test
void targetLanguageValidationTest() throws IOException {
// TODO: Uncomment C when this feature works for the C target
Target[] targets = {/*Target.C, */Target.CPP, Target.Python, Target.Rust, Target.TS};
ErrorInserter[] errorInserters = {
/*ErrorInserter.C.get(RANDOM), */ErrorInserter.CPP.get(RANDOM), ErrorInserter.PYTHON.get(RANDOM),
ErrorInserter.RUST.get(RANDOM), ErrorInserter.TYPESCRIPT.get(RANDOM)
};
for (int i = 0; i < targets.length; i++) {
checkDiagnostics(
targets[i],
alteredTest -> diagnostics -> alteredTest.getBadLines().stream().allMatch(
badLine -> {
System.out.print("Expecting an error to be reported at line " + badLine + "...");
boolean result = diagnostics.stream().anyMatch(
diagnostic -> diagnostic.getRange().getStart().getLine() == badLine
);
System.out.println(result ? " Success." : " but the expected error could not be found.");
return result;
}
),
errorInserters[i],
MAX_VALIDATIONS_PER_CATEGORY
);
}
void cppValidationTest() throws IOException {
targetLanguageValidationTest(Target.CPP, ErrorInserter.CPP.get(RANDOM));
}

/** Test for false negatives in Python validation. */
@Test
void pythonValidationTest() throws IOException {
targetLanguageValidationTest(Target.Python, ErrorInserter.PYTHON.get(RANDOM));
}

/** Test for false negatives in Rust validation. */
@Test
void rustValidationTest() throws IOException {
targetLanguageValidationTest(Target.Rust, ErrorInserter.RUST.get(RANDOM));
}

/** Test for false negatives in TypeScript validation. */
@Test
void typescriptValidationTest() throws IOException {
targetLanguageValidationTest(Target.TS, ErrorInserter.TYPESCRIPT.get(RANDOM));
}

private void targetLanguageValidationTest(Target target, ErrorInserter errorInserter) throws IOException {
checkDiagnostics(
target,
alteredTest -> diagnostics -> alteredTest.getBadLines().stream().allMatch(
badLine -> {
System.out.print("Expecting an error to be reported at line " + badLine + "...");
boolean result = diagnostics.stream().anyMatch(
diagnostic -> diagnostic.getRange().getStart().getLine() == badLine
);
System.out.println(result ? " Success." : " but the expected error could not be found.");
return result;
}
),
errorInserter,
MAX_VALIDATIONS_PER_CATEGORY
);
}

/**
Expand All @@ -108,9 +122,15 @@ private void checkDiagnostics(
LanguageServerErrorReporter.setClient(client);
for (LFTest test : selectTests(target, count)) {
client.clearDiagnostics();
AlteredTest altered = alterer == null ? null : alterer.alterTest(test.srcFile);
runTest(alterer == null ? test.srcFile : altered.getFile().toPath(), false);
Assertions.assertTrue(requirementGetter.apply(altered).test(client.getReceivedDiagnostics()));
if (alterer != null) {
try (AlteredTest altered = alterer.alterTest(test.srcFile)) {
runTest(altered.getPath(), false);
Assertions.assertTrue(requirementGetter.apply(altered).test(client.getReceivedDiagnostics()));
}
} else {
runTest(test.srcFile, false);
Assertions.assertTrue(requirementGetter.apply(null).test(client.getReceivedDiagnostics()));
}
}
}

Expand Down
4 changes: 3 additions & 1 deletion org.lflang.tests/src/org/lflang/tests/lsp/MockLanguageClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ public void publishDiagnostics(PublishDiagnosticsParams diagnostics) {
(
(d.getSeverity() == DiagnosticSeverity.Error || d.getSeverity() == DiagnosticSeverity.Warning) ?
System.err : System.out
).println("Test client received diagnostic: " + d.getMessage());
).println(
"Test client received diagnostic at line " + d.getRange().getStart().getLine() + ": " + d.getMessage()
);
}
}

Expand Down

0 comments on commit 613b9f9

Please sign in to comment.