diff --git a/.gitattributes b/.gitattributes index 05946a11..0b0f2ba8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,3 +18,6 @@ *.jar binary *.png binary +# Exclude toml-test submodule from Github language statistics. +# See https://github.com/github-linguist/linguist/blob/master/docs/overrides.md +test-multiple/src/test/resources/toml-test/* linguist-vendored diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..c2cfac2f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "toml/src/test/resources/toml-test"] + path = test-multiple/src/test/resources/toml-test + url = git@github.com:toml-lang/toml-test.git +[submodule "test-multiple/src/test/resources/toml-test"] + path = test-multiple/src/test/resources/toml-test + url = git@github.com:toml-lang/toml-test.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c09ab6d..ef8982d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,3 +62,10 @@ Execution failed for task ':japicmp'. See failure report at file:///home/guillaume/Documents/Projets/night-config/build/reports/japicmp.html ``` + +## Multi-module test suite + +The Gradle module `test-multiple` implements tests that require multiple NightConfig modules. + +In particular, it executes the tests of [the language agnostic TOML test suite](https://github.com/toml-lang/toml-test). +To avoid a test dependency on another programming language, we don't run the provided binary, but rather reimplement the tests using the same resources (valid and invalid TOML files). diff --git a/settings.gradle.kts b/settings.gradle.kts index 97213f0b..8c1c4b87 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,3 +11,4 @@ include("toml") include("yaml") include("test-shared") +include("test-multiple") diff --git a/test-multiple/build.gradle.kts b/test-multiple/build.gradle.kts new file mode 100644 index 00000000..0f33c31a --- /dev/null +++ b/test-multiple/build.gradle.kts @@ -0,0 +1,36 @@ +plugins { + `java-library` +} + +repositories { + // Use Maven Central for resolving dependencies. + mavenCentral() +} + +dependencies { + api(project(":toml")) + api(project(":json")) + testImplementation(project(":test-shared")) +} + +// Use Java 11. +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} + +// Get JUnit5 version from `libs.versions.toml`. +val versionCatalog = extensions.getByType().named("libs") +val junitVersion = versionCatalog.findVersion("junit5") + .orElseThrow{ RuntimeException("missing version in libs.versions.toml: junit5") } + .getRequiredVersion() + +// Use JUnit5. +testing { + suites { + val test by getting(JvmTestSuite::class) { + useJUnitJupiter(junitVersion) + } + } +} diff --git a/test-multiple/src/test/java/StandardTomlTests.java b/test-multiple/src/test/java/StandardTomlTests.java new file mode 100644 index 00000000..7edf172c --- /dev/null +++ b/test-multiple/src/test/java/StandardTomlTests.java @@ -0,0 +1,391 @@ +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.util.*; + +import org.junit.jupiter.api.*; + +import com.electronwill.nightconfig.core.CommentedConfig; +import com.electronwill.nightconfig.core.Config; +import com.electronwill.nightconfig.core.file.FileNotFoundAction; +import com.electronwill.nightconfig.core.io.ParsingException; +import com.electronwill.nightconfig.json.JsonParser; +import com.electronwill.nightconfig.toml.TomlFormat; +import com.electronwill.nightconfig.toml.TomlParser; +import com.electronwill.nightconfig.toml.TomlWriter; + +/** + * Executes standard toml tests from https://github.com/toml-lang/toml-test. + *

+ * The toml-test repository is cloned as a submodule, in + * {@code src/test/resources/toml-test}. + * Is contains two types of tests: + *

+ */ +public class StandardTomlTests { + + // Assumption: the current working directory is the gradle project dir + // "test-multiple". + private static final Path TOML_TEST_DIR = Path.of("src/test/resources/toml-test/tests").toAbsolutePath(); + private static final Path TEST_LIST_1_0 = TOML_TEST_DIR.resolve("files-toml-1.0.0"); + private static final Path TEST_LIST_1_1 = TOML_TEST_DIR.resolve("files-toml-1.1.0"); + + /** + * Standard valid and invalid tests (including "multi" tests) for the + * TomlParser, generated from the files in the Git submodule. + */ + @TestFactory + public List testParser() throws IOException { + var validTests = new ArrayList(); + var invalidTests = new ArrayList(); + + for (String testPath : Files.readAllLines(TEST_LIST_1_0)) { + if (testPath.startsWith("invalid/")) { + var testFile = TOML_TEST_DIR.resolve(testPath); + var testFileName = testFile.getFileName().toString(); + var relativePath = TOML_TEST_DIR.relativize(testFile); + + if (testFileName.endsWith(".toml")) { + // Regular TOML test. + invalidTests.add(dynamicTest(relativePath.toString(), () -> { + TomlParser parser = new TomlParser(); + assertThrows(Exception.class, () -> { + parser.parse(testFile, FileNotFoundAction.THROW_ERROR); + }, () -> String.format("invalid file '%s' should have been rejected by the parser", + relativePath)); + })); + + } else if (testFileName.endsWith(".multi")) { + invalidTests.add(dynamicTest(relativePath.toString(), () -> { + TomlParser parser = new TomlParser(); + + // "Multi" TOML test that contains multiple invalid key-value pairs. + for (var line : Files.readAllLines(testFile)) { + // skip blank lines and comments + if (!(line.isBlank() || line.stripLeading().startsWith("#"))) { + // we have found a key-value pair, extract the key to give a name to the test + var key = line.substring(0, line.indexOf('=')).strip(); + var testName = relativePath + "(" + key + ")"; + + System.out.println("testing " + testName); + assertThrows(ParsingException.class, () -> { + parser.parse(line); + }, () -> String.format("invalid test '%s' should have failed", testName)); + } + } + })); + } + } else if (testPath.startsWith("valid/")) { + var testFile = TOML_TEST_DIR.resolve(testPath); + var testFileName = testFile.getFileName().toString(); + var relativePath = TOML_TEST_DIR.relativize(testFile); + + if (testFileName.endsWith(".toml")) { + // Regular TOML test + JSON file containing the expected result. + var expectFile = testFile.resolveSibling(testFileName.replace(".toml", ".json")); + + validTests.add(dynamicTest(relativePath.toString(), () -> { + TomlParser tomlParser = new TomlParser(); + JsonParser jsonParser = new JsonParser(); + + try { + CommentedConfig parsed = tomlParser.parse(testFile, FileNotFoundAction.THROW_ERROR); + Config expected = jsonParser.parse(expectFile, FileNotFoundAction.THROW_ERROR); + assertConfigMatchesJsonExpectation(parsed, expected, relativePath.toString()); + } catch (Exception ex) { + fail("Exception occured in test " + relativePath, ex); + } + })); + + } + } + } + + var allTests = Arrays.asList(dynamicContainer("parser valid", validTests), dynamicContainer("parser invalid", invalidTests)); + return allTests; + } + + /** + * Standard tests for the TomlWriter, generated from the files in the Git + * submodule. + */ + @TestFactory + public List testWriter() throws IOException { + var validTests = new ArrayList(); + + for (String testPath : Files.readAllLines(TEST_LIST_1_0)) { + if (testPath.startsWith("valid/")) { + var testFile = TOML_TEST_DIR.resolve(testPath); + var testFileName = testFile.getFileName().toString(); + var relativePath = TOML_TEST_DIR.relativize(testFile); + + if (testFileName.endsWith(".json")) { + // JSON file specifying the config + TOML file containing the expected result. + var jsonFile = testFile; + var tomlFile = testFile.resolveSibling(testFileName.replace(".json", ".toml")); + + validTests.add(dynamicTest(relativePath.toString(), () -> { + TomlParser tomlParser = new TomlParser(); + TomlWriter tomlWriter = new TomlWriter(); + JsonParser jsonParser = new JsonParser(); + + try { + Config config = parseJsonExpectationToConfig( + jsonParser.parse(jsonFile, FileNotFoundAction.THROW_ERROR)); + String written = tomlWriter.writeToString(config); + + CommentedConfig writtenParsed = tomlParser.parse(written); + CommentedConfig expected = tomlParser.parse(tomlFile, FileNotFoundAction.THROW_ERROR); + assertEquals(writtenParsed, expected); + + } catch (Exception ex) { + fail("Exception occured in test " + relativePath, ex); + } + + })); + } + } + } + var allTests = Arrays.asList(dynamicContainer("writer valid", validTests)); + return allTests; + } + + private Config parseJsonExpectationToConfig(Config jsonExpect) { + return (Config) convertJsonExpectValue(jsonExpect, List.of()); + } + + @SuppressWarnings("unchecked") + private Object convertJsonExpectValue(Object jsonExpect, List key) { + if (jsonExpect instanceof List) { + var jsonList = (List) jsonExpect; + var res = new ArrayList<>(jsonList.size()); + for (var element : jsonList) { + res.add(convertJsonExpectValue(element, key)); + } + return res; + } else if (jsonExpect instanceof Config) { + var jsonConfig = (Config) jsonExpect; + if (jsonConfig.contains("type") && jsonConfig.contains("value") + && jsonConfig.get("type") instanceof String) { + // specification of a toml value: {"type": the_type, "value": string_of_value}. + String valueType = jsonConfig.get("type"); + String valueStr = jsonConfig.get("value"); + switch (valueType) { + case "float": { + switch (valueStr) { + case "+nan": + case "-nan": + case "nan": { + return Double.NaN; + } + case "+inf": + case "inf": { + return Double.POSITIVE_INFINITY; + } + case "-inf": { + return Double.NEGATIVE_INFINITY; + } + default: { + return Double.parseDouble(valueStr); + } + } + } + case "bool": { + return Boolean.parseBoolean(valueStr); + } + case "integer": { + return Long.parseLong(valueStr); + } + case "string": { + return valueStr; + } + case "datetime": { + return OffsetDateTime.parse(valueStr, FMT_DATETIME); + } + case "datetime-local": { + return LocalDateTime.parse(valueStr, FMT_DATETIME_LOCAL); + } + case "date-local": { + return LocalDate.parse(valueStr, FMT_DATE_LOCAL); + } + case "time-local": { + return LocalTime.parse(valueStr, FMT_TIME_LOCAL); + } + default: { + fail(String.format("Unknown specified value type '%s' for key %s in JSON file", valueType, + key)); + return null; + } + } + } else { + // the value is a config + Config tomlConfig = TomlFormat.newConfig(); + + for (var jsonEntry : jsonConfig.entrySet()) { + var jsonEntryKey = Collections.singletonList(jsonEntry.getKey()); + var fullKey = new ArrayList<>(key); + fullKey.add(jsonEntry.getKey()); + var jsonEntryValue = convertJsonExpectValue(jsonEntry.getValue(), fullKey); + tomlConfig.set(jsonEntryKey, jsonEntryValue); + } + + return tomlConfig; + } + } else { + fail(String.format("Invalid specified value '%s' for key %s in JSON file", jsonExpect, key)); + return null; + } + } + + private void assertConfigMatchesJsonExpectation(CommentedConfig tomlConfig, Config jsonExpect, String testName) { + // We cannot compare the configurations directly, because the expectation + // contains entries of the form {"type": the_type, "value": string_of_value}. + tomlConfig.clearComments(); // JSON has no comments, ignore them + assertMatchJsonExpectValue(List.of(), tomlConfig, jsonExpect, String.format("Valid test %s failed", testName)); + } + + private static DateTimeFormatter FMT_DATETIME = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + private static DateTimeFormatter FMT_DATETIME_LOCAL = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + private static DateTimeFormatter FMT_DATE_LOCAL = DateTimeFormatter.ISO_LOCAL_DATE; + private static DateTimeFormatter FMT_TIME_LOCAL = DateTimeFormatter.ISO_LOCAL_TIME; + + @SuppressWarnings("unchecked") + private void assertMatchJsonExpectValue(List key, Object tomlValue, Object jsonExpectValue, String msg) { + if (jsonExpectValue instanceof List) { + assertInstanceOf(List.class, tomlValue, + String.format("%s: invalid toml value for key %s: expected list", msg, key)); + List jsonList = (List) jsonExpectValue; + List tomlList = (List) tomlValue; + assertEquals(jsonList.size(), tomlList.size(), String + .format("%s: invalid toml value for key %s: expected list of size %d", msg, key, jsonList.size())); + for (int i = 0; i < tomlList.size(); i++) { + var tomlElement = tomlList.get(i); + var jsonElement = jsonList.get(i); + assertMatchJsonExpectValue(key, tomlElement, jsonElement, + String.format("%s: invalid element %d of key %s", msg, i, key)); + } + } else if (jsonExpectValue instanceof Config) { + Config jsonConfig = (Config) jsonExpectValue; + if (jsonConfig.contains("type") && jsonConfig.contains("value") + && jsonConfig.get("type") instanceof String) { + // Expect value that matches the structure: + // {"type": value_type, "value": value_string} + var expectedType = jsonConfig.get("type").toLowerCase(); + var expectedValueStr = jsonConfig.get("value"); + switch (expectedType) { + case "float": { + if (expectedValueStr.equals("+nan") || expectedValueStr.equals("-nan") + || expectedValueStr.equals("nan")) { + String err = String.format("%s: invalid value for key %s: expected NaN float, got %s", msg, + key, tomlValue); + if (tomlValue instanceof Double) { + assertTrue(Double.isNaN((double) tomlValue), err); + } else if (tomlValue instanceof Float) { + assertTrue(Float.isNaN((float) tomlValue), err); + } else { + fail(err); + } + } else if (expectedValueStr.equals("inf") || expectedValueStr.equals("+inf")) { + expectedValueStr = "Infinity"; + } else if (expectedValueStr.equals("-inf")) { + expectedValueStr = "-Infinity"; + } else { + var expectedFloat = Double.parseDouble(expectedValueStr); + assertInstanceOf(Number.class, tomlValue, + String.format("%s: invalid value for key %s, expected float", msg, key)); + assertEquals(expectedFloat, ((Number) tomlValue).doubleValue(), + String.format("%s: invalid value for key %s, expected float", msg, key)); + } + break; + } + case "bool": { + var expectedBool = Boolean.parseBoolean(String.valueOf(expectedValueStr)); + assertEquals(expectedBool, tomlValue, + String.format("%s: invalid value for key %s, expected bool", msg, key)); + break; + } + case "integer": { + var expectedInt = Long.parseLong(String.valueOf(expectedValueStr)); + assertInstanceOf(Number.class, tomlValue, + String.format("%s: invalid value for key %s, expected integer", msg, key)); + assertEquals(expectedInt, ((Number) tomlValue).longValue(), + String.format("%s: invalid value for key %s, expected integer", msg, key)); + break; + } + case "string": { + var expectedString = expectedValueStr; + assertInstanceOf(String.class, tomlValue, + String.format("%s: invalid value for key %s, expected string", msg, key)); + assertEquals(expectedString, tomlValue, + String.format("%s: invalid value for key %s, expected string", msg, key)); + break; + } + case "datetime": { + var expectedDateTime = OffsetDateTime.parse(expectedValueStr, FMT_DATETIME); + assertEquals(expectedDateTime, tomlValue, + String.format("%s: invalid value for key %s, expected datetime", msg, key)); + break; + } + case "datetime-local": { + var expectedDateTime = LocalDateTime.parse(expectedValueStr, FMT_DATETIME_LOCAL); + assertEquals(expectedDateTime, tomlValue, + String.format("%s: invalid value for key %s, expected datetime-local", msg, key)); + break; + } + case "date-local": { + var expectedDateTime = LocalDate.parse(expectedValueStr, FMT_DATE_LOCAL); + assertEquals(expectedDateTime, tomlValue, + String.format("%s: invalid value for key %s, expected date-local", msg, key)); + break; + } + case "time-local": { + var expectedDateTime = LocalTime.parse(expectedValueStr, FMT_TIME_LOCAL); + assertEquals(expectedDateTime, tomlValue, + String.format("%s: invalid value for key %s, expected time-local", msg, key)); + break; + } + default: { + fail(String.format("Unknown expected value type '%s' for key '%s' in JSON file", expectedType, + key)); + } + } + } else { + // expect config + assertInstanceOf(Config.class, tomlValue, + String.format("%s: invalid toml value for key %s: expected config", msg, key)); + Config tomlConfig = (Config) tomlValue; + assertEquals(jsonConfig.size(), tomlConfig.size(), String.format( + "%s: invalid toml value for key %s: expected config of size %d", msg, key, jsonConfig.size())); + for (var jsonEntry : jsonConfig.entrySet()) { + var jsonEntryValue = jsonEntry.getRawValue(); + var tomlEntryValue = tomlConfig.getRaw(Collections.singletonList(jsonEntry.getKey())); + var fullKey = new ArrayList<>(key); + fullKey.add(jsonEntry.getKey()); + assertMatchJsonExpectValue(fullKey, tomlEntryValue, jsonEntryValue, msg); + } + } + } else { + fail(String.format("Invalid expectation in JSON file: %s should be a list of config", key)); + } + } + +} diff --git a/test-multiple/src/test/resources/toml-test b/test-multiple/src/test/resources/toml-test new file mode 160000 index 00000000..bd041f0b --- /dev/null +++ b/test-multiple/src/test/resources/toml-test @@ -0,0 +1 @@ +Subproject commit bd041f0b741351cf146296ec060e5e3e452e6468