From f8a966d364ddb6b868681ca55dbd79217ab786fc Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:35:42 -0400 Subject: [PATCH 01/28] add base classes, main broken --- .../gtfsvalidator/table/AnyTableLoader.java | 18 +-- .../gtfsvalidator/table/GtfsContainer.java | 50 ++++++++ .../gtfsvalidator/table/GtfsDescriptor.java | 23 ++++ .../table/GtfsFeedContainer.java | 21 ++-- .../gtfsvalidator/table/GtfsFeedLoader.java | 9 +- .../table/GtfsTableContainer.java | 47 +------- .../table/GtfsTableDescriptor.java | 22 +--- .../gtfsvalidator/table/TableStatus.java | 35 ++++++ .../gtfsvalidator/testing/LoadingHelper.java | 2 +- .../validator/DefaultValidatorProvider.java | 15 ++- .../validator/ValidatorLoader.java | 26 ++--- .../validator/ValidatorProvider.java | 11 +- .../table/AnyTableLoaderTest.java | 18 +-- .../table/GtfsFeedContainerTest.java | 1 - .../testgtfs/GtfsTestTableContainer.java | 4 +- .../testgtfs/GtfsTestTableContainer2.java | 4 +- .../testgtfs/GtfsTestTableDescriptor.java | 3 +- .../testgtfs/GtfsTestTableDescriptor2.java | 3 +- .../DefaultValidatorProviderTest.java | 2 +- .../validator/ValidatorLoaderTest.java | 2 +- .../report/model/FeedMetadata.java | 20 ++-- .../report/model/TableMetadata.java | 12 +- .../validator/DateTripsValidatorTest.java | 2 +- .../ExpiredCalendarValidatorTest.java | 2 +- ...atchingFeedAndAgencyLangValidatorTest.java | 2 +- ...gCalendarAndCalendarDateValidatorTest.java | 2 +- .../NetworkIdConsistencyValidatorTest.java | 5 +- .../processor/notices/NoticesProcessor.java | 46 ++++---- .../processor/GtfsAnnotationProcessor.java | 3 + .../processor/TableContainerGenerator.java | 12 +- .../processor/TableDescriptorGenerator.java | 8 +- .../tests/CurrencyAmountSchemaTest.java | 84 ------------- .../processor/tests/MixedCaseSchemaTest.java | 110 ------------------ 33 files changed, 236 insertions(+), 388 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java delete mode 100644 processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java delete mode 100644 processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java index 3765bf22cb..526c539e4a 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java @@ -61,13 +61,11 @@ public static GtfsTableContainer load( csvFile = new CsvFile(csvInputStream, gtfsFilename, settings); } catch (TextParsingException e) { noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e)); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS); } if (csvFile.isEmpty()) { noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename)); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.EMPTY_FILE); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE); } final CsvHeader header = csvFile.getHeader(); final ImmutableList columnDescriptors = tableDescriptor.getColumns(); @@ -75,8 +73,7 @@ public static GtfsTableContainer load( validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors); noticeContainer.addAll(headerNotices); if (headerNotices.hasValidationErrors()) { - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS); } final int nColumns = columnDescriptors.size(); final ImmutableMap fieldLoadersMap = tableDescriptor.getFieldLoaders(); @@ -133,15 +130,13 @@ public static GtfsTableContainer load( } } catch (TextParsingException e) { noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e)); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); } finally { logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors); } if (hasUnparsableRows) { logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); } GtfsTableContainer table = tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer); @@ -203,8 +198,7 @@ public static GtfsTableContainer loadMissingFile( NoticeContainer noticeContainer) { String gtfsFilename = tableDescriptor.gtfsFilename(); GtfsTableContainer table = - tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.MISSING_FILE); + tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); if (tableDescriptor.isRecommended()) { noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java new file mode 100644 index 0000000000..c5ab7ce385 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java @@ -0,0 +1,50 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.util.List; +import java.util.Optional; + +public abstract class GtfsContainer { + + private final D descriptor; + private final TableStatus tableStatus; + + public GtfsContainer(D descriptor, TableStatus tableStatus) { + this.tableStatus = tableStatus; + this.descriptor = descriptor; + } + + public TableStatus getTableStatus() { + return tableStatus; + } + + public D getDescriptor() { + return descriptor; + } + + public abstract Class getEntityClass(); + + public int entityCount() { + return getEntities().size(); + } + + public abstract List getEntities(); + + public abstract String gtfsFilename(); + + public abstract Optional byTranslationKey(String recordId, String recordSubId); + + public boolean isMissingFile() { + return tableStatus == TableStatus.MISSING_FILE; + } + + public boolean isParsedSuccessfully() { + switch (tableStatus) { + case PARSABLE_HEADERS_AND_ROWS: + return true; + case MISSING_FILE: + return !descriptor.isRequired(); + default: + return false; + } + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java new file mode 100644 index 0000000000..bdcbcaffb5 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java @@ -0,0 +1,23 @@ +package org.mobilitydata.gtfsvalidator.table; + +// TODO: review class name maybe GtfsFileDescriptor +public abstract class GtfsDescriptor { + // True if the specified file is required in a feed. + private boolean required; + + private TableStatus tableStatus; + + public abstract boolean isRecommended(); + + public abstract Class getEntityClass(); + + public abstract String gtfsFilename(); + + public boolean isRequired() { + return this.required; + } + + public void setRequired(boolean required) { + this.required = required; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java index 9b96d3d68d..d05deb0f7f 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java @@ -18,20 +18,19 @@ import com.google.common.base.Ascii; import java.util.*; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; /** * Container for a whole parsed GTFS feed with all its tables. * - *

The tables are kept as {@code GtfsTableContainer} instances. + *

The tables are kept as {@code GtfsContainer} instances. */ public class GtfsFeedContainer { - private final Map> tables = new HashMap<>(); - private final Map, GtfsTableContainer> tablesByClass = + private final Map> tables = new HashMap<>(); + private final Map, GtfsContainer> tablesByClass = new HashMap<>(); - public GtfsFeedContainer(List> tableContainerList) { - for (GtfsTableContainer table : tableContainerList) { + public GtfsFeedContainer(List> tableContainerList) { + for (GtfsContainer table : tableContainerList) { tables.put(table.gtfsFilename(), table); tablesByClass.put(table.getClass(), table); } @@ -49,11 +48,11 @@ public GtfsFeedContainer(List> tableContainerList) { * @param filename file name, including ".txt" extension * @return GTFS table or empty if the table is not supported by schema */ - public Optional> getTableForFilename(String filename) { + public Optional> getTableForFilename(String filename) { return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null)); } - public > T getTable(Class clazz) { + public > T getTable(Class clazz) { return (T) tablesByClass.get(clazz); } @@ -65,7 +64,7 @@ public > T getTable(Class clazz) { * @return true if all files were successfully parsed, false otherwise */ public boolean isParsedSuccessfully() { - for (GtfsTableContainer table : tables.values()) { + for (GtfsContainer table : tables.values()) { if (!table.isParsedSuccessfully()) { return false; } @@ -73,13 +72,13 @@ public boolean isParsedSuccessfully() { return true; } - public Collection> getTables() { + public Collection> getTables() { return tables.values(); } public String tableTotalsText() { List totalList = new ArrayList<>(); - for (GtfsTableContainer table : tables.values()) { + for (GtfsContainer table : tables.values()) { if (table.getTableStatus() == TableStatus.MISSING_FILE && !table.getDescriptor().isRequired()) { continue; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 1edb051184..a4586b6ead 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -107,7 +107,7 @@ public GtfsFeedContainer loadAndValidate( loaderCallables.add( () -> { NoticeContainer loaderNotices = new NoticeContainer(); - GtfsTableContainer tableContainer; + GtfsContainer tableContainer; try (InputStream inputStream = gtfsInput.getFile(filename)) { try { tableContainer = @@ -130,7 +130,7 @@ public GtfsFeedContainer loadAndValidate( }); } } - ArrayList> tableContainers = new ArrayList<>(); + ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); for (GtfsTableDescriptor tableDescriptor : remainingDescriptors.values()) { tableContainers.add( @@ -186,11 +186,10 @@ private static void addThreadExecutionError( } static class TableAndNoticeContainers { - final GtfsTableContainer tableContainer; + final GtfsContainer tableContainer; final NoticeContainer noticeContainer; - public TableAndNoticeContainers( - GtfsTableContainer tableContainer, NoticeContainer noticeContainer) { + public TableAndNoticeContainers(GtfsContainer tableContainer, NoticeContainer noticeContainer) { this.tableContainer = tableContainer; this.noticeContainer = noticeContainer; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java index 010da6a1da..e6d1114a92 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java @@ -31,25 +31,22 @@ * * @param subclass of {@code GtfsEntity} */ -public abstract class GtfsTableContainer { +public abstract class GtfsTableContainer + extends GtfsContainer { - private final GtfsTableDescriptor descriptor; + private final D descriptor; private final TableStatus tableStatus; private final CsvHeader header; - public GtfsTableContainer( - GtfsTableDescriptor descriptor, TableStatus tableStatus, CsvHeader header) { + public GtfsTableContainer(D descriptor, TableStatus tableStatus, CsvHeader header) { + super(descriptor, tableStatus); this.descriptor = descriptor; this.tableStatus = tableStatus; this.header = header; } - public GtfsTableDescriptor getDescriptor() { - return descriptor; - } - public TableStatus getTableStatus() { return tableStatus; } @@ -130,38 +127,4 @@ public boolean isParsedSuccessfully() { return false; } } - - /** - * Status of loading this table. This is includes parsing of the CSV file and validation of the - * single file, but does not include any cross-file validations. - */ - public enum TableStatus { - /** The file is completely empty, i.e. it has no rows and even no headers. */ - EMPTY_FILE, - - /** The file is missing in the GTFS feed. */ - MISSING_FILE, - - /** The file was parsed successfully. It has headers and 0, 1 or many rows. */ - PARSABLE_HEADERS_AND_ROWS, - - /** - * The file has invalid headers, e.g., they failed to parse or some required headers are - * missing. The other rows were not scanned. - * - *

Note that unknown headers are not considered invalid. - */ - INVALID_HEADERS, - - /** - * Some of the rows failed to parse, e.g., they have missing required fields or invalid field - * values. - * - *

However, the headers are valid. - * - *

This does not include cross-file or cross-row validation. This also does not include - * single-entity validation. - */ - UNPARSABLE_ROWS, - } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java index 01f3b773db..e448db4972 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java @@ -7,35 +7,17 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; -public abstract class GtfsTableDescriptor { +public abstract class GtfsTableDescriptor extends GtfsDescriptor { - // True if the specified file is required in a feed. - private boolean required; - - public abstract GtfsTableContainer createContainerForInvalidStatus( - GtfsTableContainer.TableStatus tableStatus); + public abstract GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus); public abstract GtfsTableContainer createContainerForHeaderAndEntities( CsvHeader header, List entities, NoticeContainer noticeContainer); public abstract GtfsEntityBuilder createEntityBuilder(); - public abstract Class getEntityClass(); - - public abstract String gtfsFilename(); - public abstract ImmutableMap getFieldLoaders(); - public abstract boolean isRecommended(); - - public boolean isRequired() { - return this.required; - } - - public void setRequired(boolean required) { - this.required = required; - } - public abstract Optional maxCharsPerColumn(); public abstract ImmutableList getColumns(); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java new file mode 100644 index 0000000000..417d01e772 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java @@ -0,0 +1,35 @@ +package org.mobilitydata.gtfsvalidator.table; + +/** + * Status of loading this table. This is includes parsing of the CSV file and validation of the + * single file, but does not include any cross-file validations. + */ +public enum TableStatus { + /** The file is completely empty, i.e. it has no rows and even no headers. */ + EMPTY_FILE, + + /** The file is missing in the GTFS feed. */ + MISSING_FILE, + + /** The file was parsed successfully. It has headers and 0, 1 or many rows. */ + PARSABLE_HEADERS_AND_ROWS, + + /** + * The file has invalid headers, e.g., they failed to parse or some required headers are missing. + * The other rows were not scanned. + * + *

Note that unknown headers are not considered invalid. + */ + INVALID_HEADERS, + + /** + * Some of the rows failed to parse, e.g., they have missing required fields or invalid field + * values. + * + *

However, the headers are valid. + * + *

This does not include cross-file or cross-row validation. This also does not include + * single-entity validation. + */ + UNPARSABLE_ROWS, +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java index b51f788eb6..1fefbfaa39 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java @@ -58,7 +58,7 @@ public void setValidatorLoader(ValidatorLoader validatorLoader) { this.validatorLoader = validatorLoader; } - public > Y load( + public > Y load( GtfsTableDescriptor tableDescriptor, String... lines) throws ValidatorLoaderException { String content = Arrays.stream(lines).collect(Collectors.joining("\n")); InputStream in = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java index 835e494d3b..3628f844fc 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java @@ -21,9 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; -import org.mobilitydata.gtfsvalidator.table.GtfsEntity; -import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.*; import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader.ValidatorWithDependencyStatus; /** Default implementation of {@link ValidatorProvider}. */ @@ -35,7 +33,7 @@ public class DefaultValidatorProvider implements ValidatorProvider { private final TableHeaderValidator tableHeaderValidator; private final ListMultimap, Class>> singleEntityValidators; - private final ListMultimap>, Class> + private final ListMultimap>, Class> singleFileValidators; private final List> multiFileValidators; @@ -103,12 +101,13 @@ public List> createSingleEntityV @Override @SuppressWarnings("unchecked") - public List createSingleFileValidators( - GtfsTableContainer table, - Consumer> validatorsWithParsingErrors) { + public + List createSingleFileValidators( + GtfsTableContainer table, + Consumer> validatorsWithParsingErrors) { List validators = new ArrayList<>(); for (Class validatorClass : - singleFileValidators.get((Class>) table.getClass())) { + singleFileValidators.get((Class>) table.getClass())) { try { ValidatorWithDependencyStatus validatorWithStatus = ValidatorLoader.createSingleFileValidator(validatorClass, table, validationContext); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java index edd3ffb0a8..43e089d569 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java @@ -29,9 +29,9 @@ import javax.annotation.Nullable; import javax.inject.Inject; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsContainer; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; /** * A {@code ValidatorLoader} object locates all validators registered with {@code @GtfsValidator} @@ -43,7 +43,7 @@ public class ValidatorLoader { private final ListMultimap, Class>> singleEntityValidators = ArrayListMultimap.create(); - private final ListMultimap>, Class> + private final ListMultimap>, Class> singleFileValidators = ArrayListMultimap.create(); private final List> multiFileValidators = new ArrayList<>(); @@ -75,7 +75,7 @@ private ValidatorLoader() {} } /** Loaded single-file validator classes keyed by table container class. */ - public ListMultimap>, Class> + public ListMultimap>, Class> getSingleFileValidators() { return singleFileValidators; } @@ -113,14 +113,14 @@ private void addFileValidator(Class validatorClass) // Indicates that the full GtfsFeedContainer needs to be injected. boolean injectFeedContainer = false; // Find out which GTFS tables need to be injected. - List>> injectedTables = new ArrayList<>(); + List>> injectedTables = new ArrayList<>(); for (Class parameterType : constructor.getParameterTypes()) { if (GtfsFeedContainer.class.isAssignableFrom(parameterType)) { injectFeedContainer = true; continue; } - if (GtfsTableContainer.class.isAssignableFrom(parameterType)) { - injectedTables.add((Class>) parameterType); + if (GtfsContainer.class.isAssignableFrom(parameterType)) { + injectedTables.add((Class>) parameterType); } } @@ -201,7 +201,7 @@ ValidatorWithDependencyStatus createValidatorWithContext( public static ValidatorWithDependencyStatus createSingleFileValidator( Class clazz, - GtfsTableContainer table, + GtfsContainer table, ValidationContext validationContext) throws ReflectiveOperationException, ValidatorLoaderException { return (ValidatorWithDependencyStatus) @@ -222,7 +222,7 @@ public static ValidatorWithDependencyStatus createM */ private static class DependencyResolver { private final ValidationContext context; - @Nullable private final GtfsTableContainer tableContainer; + @Nullable private final GtfsContainer tableContainer; @Nullable private final GtfsFeedContainer feedContainer; /** This will be set to true if a resolved dependency was not parsed successfully. */ @@ -230,7 +230,7 @@ private static class DependencyResolver { public DependencyResolver( ValidationContext context, - @Nullable GtfsTableContainer tableContainer, + @Nullable GtfsContainer tableContainer, @Nullable GtfsFeedContainer feedContainer) { this.context = context; this.tableContainer = tableContainer; @@ -257,9 +257,9 @@ public Object resolveDependency(Class parameterClass) { } return tableContainer; } - if (feedContainer != null && GtfsTableContainer.class.isAssignableFrom(parameterClass)) { - GtfsTableContainer container = - feedContainer.getTable((Class>) parameterClass); + if (feedContainer != null && GtfsContainer.class.isAssignableFrom(parameterClass)) { + GtfsContainer container = + feedContainer.getTable((Class>) parameterClass); if (container != null && !container.isParsedSuccessfully()) { dependenciesHaveErrors = true; } @@ -305,7 +305,7 @@ public String listValidators() { if (!singleFileValidators.isEmpty()) { builder.append("Single-file validators\n"); for (Map.Entry< - Class>, Collection>> + Class>, Collection>> entry : singleFileValidators.asMap().entrySet()) { builder.append("\t").append(entry.getKey().getSimpleName()).append(": "); for (Class validatorClass : entry.getValue()) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java index 2c243a8dc9..84ea9880bb 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java @@ -18,9 +18,7 @@ import java.util.List; import java.util.function.Consumer; -import org.mobilitydata.gtfsvalidator.table.GtfsEntity; -import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.*; /** * Provider of all kinds of validators for fields, entities and files. @@ -57,9 +55,10 @@ List> createSingleEntityValidato * @param table GTFS table to validate * @param type of the GTFS entity */ - List createSingleFileValidators( - GtfsTableContainer table, - Consumer> validatorsWithParsingErrors); + + List createSingleFileValidators( + GtfsTableContainer table, + Consumer> validatorsWithParsingErrors); /** * Creates a list of cross-table validators. Any validator that has a dependency with parse errors diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java index 6c6da42f6e..5dee4d9aca 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java @@ -46,8 +46,7 @@ public void setup() { public void invalidInputStream() { var testTableDescriptor = mock(GtfsTableDescriptor.class); when(testTableDescriptor.gtfsFilename()).thenReturn("_not_a_valid_file_"); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS)) .thenReturn(mockContainer); var loadedContainer = @@ -61,8 +60,7 @@ public void invalidInputStream() { public void emptyInputStream() { var testTableDescriptor = mock(GtfsTableDescriptor.class); when(testTableDescriptor.gtfsFilename()).thenReturn("filename"); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.EMPTY_FILE)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE)) .thenReturn(mockContainer); InputStream csvInputStream = toInputStream(""); @@ -79,8 +77,7 @@ public void invalidHeaders() { var testTableDescriptor = mock(GtfsTableDescriptor.class); when(testTableDescriptor.gtfsFilename()).thenReturn("filename"); when(testTableDescriptor.getColumns()).thenReturn(ImmutableList.of()); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS)) .thenReturn(mockContainer); InputStream csvInputStream = toInputStream("A file with no headers"); ValidationNotice headerValidationNotice = new EmptyColumnNameNotice("stops.txt", 0); @@ -109,8 +106,7 @@ public void validate( @Test public void invalidRowLengthNotice() { var testTableDescriptor = spy(new GtfsTestTableDescriptor()); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS)) .thenReturn(mockContainer); when(validatorProvider.getTableHeaderValidator()).thenReturn(mock(TableHeaderValidator.class)); InputStream inputStream = toInputStream("id,code\n" + "s1\n"); @@ -138,8 +134,7 @@ public void parsableTableRows() { var loadedContainer = AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); - assertThat(loadedContainer.getTableStatus()) - .isEqualTo(GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS); + assertThat(loadedContainer.getTableStatus()).isEqualTo(TableStatus.PARSABLE_HEADERS_AND_ROWS); verify(validator, times(1)).validate(any()); } @@ -165,8 +160,7 @@ public void missingRequiredField() { .setIsMixedCase(false) .setIsCached(false) .build())); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS)) .thenReturn(mockContainer); when(validatorProvider.getTableHeaderValidator()).thenReturn(mock(TableHeaderValidator.class)); when(validatorProvider.getFieldValidator()).thenReturn(mock(GtfsFieldValidator.class)); diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java index 21735bac2b..15073b5d4f 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableList; import org.junit.Test; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestTableContainer; public class GtfsFeedContainerTest { diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java index e84ab8041a..94940cced1 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java @@ -23,8 +23,10 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; -public class GtfsTestTableContainer extends GtfsTableContainer { +public class GtfsTestTableContainer + extends GtfsTableContainer { private static final ImmutableList KEY_COLUMN_NAMES = ImmutableList.of(GtfsTestEntity.ID_FIELD_NAME); diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java index 962b5bb6e7..662adc7319 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java @@ -23,9 +23,11 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; // We need a second test table class to test multi file validators. -public class GtfsTestTableContainer2 extends GtfsTableContainer { +public class GtfsTestTableContainer2 + extends GtfsTableContainer { private static final ImmutableList KEY_COLUMN_NAMES = ImmutableList.of(GtfsTestEntity.ID_FIELD_NAME); diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java index 58dcdae890..2da7a405a9 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java @@ -13,8 +13,7 @@ public class GtfsTestTableDescriptor extends GtfsTableDescriptor { @Override - public GtfsTableContainer createContainerForInvalidStatus( - GtfsTableContainer.TableStatus tableStatus) { + public GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus) { return new GtfsTestTableContainer(tableStatus); } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java index c8442428a5..a3896d7237 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java @@ -14,8 +14,7 @@ // We need a second test table descriptor to test multi file contaioners public class GtfsTestTableDescriptor2 extends GtfsTableDescriptor { @Override - public GtfsTableContainer createContainerForInvalidStatus( - GtfsTableContainer.TableStatus tableStatus) { + public GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus) { return new GtfsTestTableContainer2(tableStatus); } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java index 90d10fa10f..823234582e 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java @@ -11,7 +11,7 @@ import org.junit.runners.JUnit4; import org.mobilitydata.gtfsvalidator.TestUtils; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestEntity; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestEntityValidator; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestMultiFileValidator; diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java index c169d29ac3..eadc3983d0 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java @@ -24,7 +24,7 @@ import org.mobilitydata.gtfsvalidator.input.CountryCode; import org.mobilitydata.gtfsvalidator.input.DateForValidation; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestEntityValidator; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestSingleFileValidator; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestTableContainer; diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index 832daea602..fc9c1d6aba 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -78,12 +78,10 @@ public static FeedMetadata from(GtfsFeedContainer feedContainer, ImmutableSet) - feedContainer.getTableForFilename(GtfsFeedInfo.FILENAME).get()); + (GtfsContainer) feedContainer.getTableForFilename(GtfsFeedInfo.FILENAME).get()); } feedMetadata.loadAgencyData( - (GtfsTableContainer) - feedContainer.getTableForFilename(GtfsAgency.FILENAME).get()); + (GtfsContainer) feedContainer.getTableForFilename(GtfsAgency.FILENAME).get()); feedMetadata.loadSpecFeatures(feedContainer); return feedMetadata; } @@ -102,7 +100,7 @@ private void setCounts(GtfsFeedContainer feedContainer) { setCount(COUNTS_BLOCKS, feedContainer, GtfsTrip.FILENAME, GtfsTrip.class, GtfsTrip::blockId); } - private , E extends GtfsEntity> void setCount( + private void setCount( String countName, GtfsFeedContainer feedContainer, String fileName, @@ -112,13 +110,11 @@ private , E extends GtfsEntity> void setCount( var table = feedContainer.getTableForFilename(fileName); this.counts.put( countName, - table - .map(gtfsTableContainer -> loadUniqueCount(gtfsTableContainer, clazz, idExtractor)) - .orElse(0)); + table.map(GtfsContainer -> loadUniqueCount(GtfsContainer, clazz, idExtractor)).orElse(0)); } private int loadUniqueCount( - GtfsTableContainer table, Class clazz, Function idExtractor) { + GtfsContainer table, Class clazz, Function idExtractor) { // Iterate through entities and count unique IDs Set uniqueIds = new HashSet<>(); for (GtfsEntity entity : table.getEntities()) { @@ -296,13 +292,13 @@ private void loadRouteColorsFeature(GtfsFeedContainer feedContainer) { List.of((Function) GtfsRoute::hasRouteTextColor))); } - private void loadAgencyData(GtfsTableContainer agencyTable) { + private void loadAgencyData(GtfsContainer agencyTable) { for (GtfsAgency agency : agencyTable.getEntities()) { agencies.add(AgencyMetadata.from(agency)); } } - private void loadFeedInfo(GtfsTableContainer feedTable) { + private void loadFeedInfo(GtfsContainer feedTable) { var info = feedTable.getEntities().isEmpty() ? null : feedTable.getEntities().get(0); feedInfo.put(FEED_INFO_PUBLISHER_NAME, info == null ? "N/A" : info.feedPublisherName()); @@ -347,7 +343,7 @@ private boolean hasAtLeastOneRecordForFields( public ArrayList foundFiles() { var foundFiles = new ArrayList(); for (var table : tableMetaData.values()) { - if (table.getTableStatus() != GtfsTableContainer.TableStatus.MISSING_FILE) { + if (table.getTableStatus() != TableStatus.MISSING_FILE) { foundFiles.add(table.getFilename()); } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java index 5cddcccb5f..e1c1dc7ba4 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java @@ -1,21 +1,21 @@ package org.mobilitydata.gtfsvalidator.report.model; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; public class TableMetadata { private final String filename; - private final GtfsTableContainer.TableStatus tableStatus; + private final TableStatus tableStatus; private final int entityCount; - public TableMetadata( - String filename, GtfsTableContainer.TableStatus tableStatus, int entityCount) { + public TableMetadata(String filename, TableStatus tableStatus, int entityCount) { this.filename = filename; this.tableStatus = tableStatus; this.entityCount = entityCount; } - public static TableMetadata from(GtfsTableContainer table) { + public static TableMetadata from(GtfsContainer table) { return new TableMetadata(table.gtfsFilename(), table.getTableStatus(), table.entityCount()); } @@ -23,7 +23,7 @@ public String getFilename() { return filename; } - public GtfsTableContainer.TableStatus getTableStatus() { + public TableStatus getTableStatus() { return tableStatus; } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java index 1a8e367351..3b42d555d6 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java @@ -16,7 +16,7 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.table.*; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.type.GtfsDate; import org.mobilitydata.gtfsvalidator.util.CalendarUtilTest; diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java index eac18ce095..2ad7a1fe0c 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java @@ -30,7 +30,7 @@ import org.mobilitydata.gtfsvalidator.input.DateForValidation; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.table.*; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.type.GtfsDate; @RunWith(JUnit4.class) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java index bfeef3dc7c..6bdd7750c9 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java @@ -31,7 +31,7 @@ import org.mobilitydata.gtfsvalidator.table.GtfsAgencyTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedInfo; import org.mobilitydata.gtfsvalidator.table.GtfsFeedInfoTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.validator.MatchingFeedAndAgencyLangValidator.FeedInfoLangAndAgencyLangMismatchNotice; @RunWith(JUnit4.class) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java index afb6e585c1..f9621bb995 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java @@ -31,7 +31,7 @@ import org.mobilitydata.gtfsvalidator.table.GtfsCalendarDate; import org.mobilitydata.gtfsvalidator.table.GtfsCalendarDateTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsCalendarTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.type.GtfsDate; import org.mobilitydata.gtfsvalidator.util.CalendarUtilTest; import org.mobilitydata.gtfsvalidator.validator.MissingCalendarAndCalendarDateValidator.MissingCalendarAndCalendarDateFilesNotice; diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java index 0dfe65a6d2..baf0358156 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java @@ -28,10 +28,9 @@ public void setup() { noticeContainer); routeNetworkTableContainer = new GtfsRouteNetworkTableContainer( - new GtfsRouteNetworkTableDescriptor(), GtfsTableContainer.TableStatus.MISSING_FILE); + new GtfsRouteNetworkTableDescriptor(), TableStatus.MISSING_FILE); networkTableContainer = - new GtfsNetworkTableContainer( - new GtfsNetworkTableDescriptor(), GtfsTableContainer.TableStatus.MISSING_FILE); + new GtfsNetworkTableContainer(new GtfsNetworkTableDescriptor(), TableStatus.MISSING_FILE); } @Test diff --git a/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java b/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java index 5b489a1822..0211b88787 100644 --- a/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java +++ b/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java @@ -52,31 +52,35 @@ public SourceVersion getSupportedSourceVersion() { @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { - for (TypeElement element : - typesIn(roundEnv.getElementsAnnotatedWith(GtfsValidationNotice.class))) { - Optional comments = docCommentsFactory.create(element); - if (comments.isEmpty()) { - continue; - } + try { + for (TypeElement element : + typesIn(roundEnv.getElementsAnnotatedWith(GtfsValidationNotice.class))) { + Optional comments = docCommentsFactory.create(element); + if (comments.isEmpty()) { + continue; + } - PackageElement packageElement = processingEnv.getElementUtils().getPackageOf(element); - String resourceName = NoticeDocComments.getResourceNameForTypeElement(element); + PackageElement packageElement = processingEnv.getElementUtils().getPackageOf(element); + String resourceName = NoticeDocComments.getResourceNameForTypeElement(element); - try { - FileObject resource = - processingEnv - .getFiler() - .createResource( - StandardLocation.CLASS_OUTPUT, - packageElement.getQualifiedName(), - resourceName, - element); - try (Writer writer = resource.openWriter()) { - GSON.toJson(comments.get(), writer); + try { + FileObject resource = + processingEnv + .getFiler() + .createResource( + StandardLocation.CLASS_OUTPUT, + packageElement.getQualifiedName(), + resourceName, + element); + try (Writer writer = resource.openWriter()) { + GSON.toJson(comments.get(), writer); + } + } catch (IOException e) { + throw new RuntimeException(e); } - } catch (IOException e) { - throw new RuntimeException(e); } + } catch (Exception e) { + System.out.println("Error: " + e); } return false; } diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java index afa907421a..ce1c4504cc 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java @@ -51,6 +51,9 @@ public class GtfsAnnotationProcessor extends AbstractProcessor { private final Analyser analyser = new Analyser(); + public GtfsAnnotationProcessor() { + super(); + } /** * Sanitizes the result of {@link RoundEnvironment#getElementsAnnotatedWith}, which otherwise can * contain elements annotated with annotations of ERROR type. diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java index f215ecdf0a..2dc5735698 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java @@ -32,6 +32,7 @@ import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.TableStatus; /** * Generates code for a container for a loaded GTFS table. @@ -64,7 +65,12 @@ public TypeSpec generateGtfsContainerClass() { TypeSpec.Builder typeSpec = TypeSpec.classBuilder(classNames.tableContainerSimpleName()) .superclass( - ParameterizedTypeName.get(ClassName.get(GtfsTableContainer.class), gtfsEntityType)) + ParameterizedTypeName.get( + ClassName.get(GtfsTableContainer.class), + classNames.entityImplementationTypeName(), + ParameterizedTypeName.get( + ClassName.get(GtfsTableDescriptor.class), + classNames.entityImplementationTypeName()))) .addAnnotation(Generated.class) .addModifiers(Modifier.PUBLIC, Modifier.FINAL); @@ -126,7 +132,7 @@ private MethodSpec generateConstructorWithStatus() { return MethodSpec.constructorBuilder() .addModifiers(Modifier.PUBLIC) .addParameter(tableDescriptorType, "descriptor") - .addParameter(GtfsTableContainer.TableStatus.class, "tableStatus") + .addParameter(TableStatus.class, "tableStatus") .addStatement("super(descriptor, tableStatus, $T.EMPTY)", CsvHeader.class) .addStatement("this.entities = new $T<>()", ArrayList.class) .build(); @@ -182,7 +188,7 @@ private MethodSpec generateForStatusMethod() { "Creates a table with the given TableStatus. This method is intended to be" + " used in tests.") .addModifiers(Modifier.PUBLIC, Modifier.STATIC) - .addParameter(GtfsTableContainer.TableStatus.class, "tableStatus") + .addParameter(TableStatus.class, "tableStatus") .addStatement( "return new $T(new $T(), tableStatus)", tableContainerTypeName, diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java index 514bbe98c9..187f7fbc71 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java @@ -42,11 +42,7 @@ import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.parsing.FieldCache; import org.mobilitydata.gtfsvalidator.parsing.RowParser; -import org.mobilitydata.gtfsvalidator.table.GtfsColumnDescriptor; -import org.mobilitydata.gtfsvalidator.table.GtfsEntityBuilder; -import org.mobilitydata.gtfsvalidator.table.GtfsFieldLoader; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.*; /** * Generates code for a GtfsTableDescriptor subclass for a specific GTFS table. @@ -149,7 +145,7 @@ private MethodSpec generateCreateContainerForInvalidStatusMethod() { return MethodSpec.methodBuilder("createContainerForInvalidStatus") .addAnnotation(Override.class) .addModifiers(Modifier.PUBLIC) - .addParameter(GtfsTableContainer.TableStatus.class, "tableStatus") + .addParameter(TableStatus.class, "tableStatus") .returns(GtfsTableContainer.class) .addStatement("return new $T(this, tableStatus)", classNames.tableContainerTypeName()) .build(); diff --git a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java deleted file mode 100644 index a2eafc802f..0000000000 --- a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.mobilitydata.gtfsvalidator.processor.tests; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.common.collect.ImmutableList; -import java.math.BigDecimal; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mobilitydata.gtfsvalidator.notice.InvalidCurrencyAmountNotice; -import org.mobilitydata.gtfsvalidator.table.CurrencyAmountTableDescriptor; -import org.mobilitydata.gtfsvalidator.testing.LoadingHelper; -import org.mobilitydata.gtfsvalidator.validator.CurrencyAmountCurrencyAmountValidator; -import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader; -import org.mobilitydata.gtfsvalidator.validator.ValidatorLoaderException; - -@RunWith(JUnit4.class) -public class CurrencyAmountSchemaTest { - - private CurrencyAmountTableDescriptor tableDescriptor; - private LoadingHelper helper; - - @Before - public void setup() throws ValidatorLoaderException { - tableDescriptor = new CurrencyAmountTableDescriptor(); - helper = new LoadingHelper(); - helper.setValidatorLoader( - ValidatorLoader.createForClasses( - ImmutableList.of(CurrencyAmountCurrencyAmountValidator.class))); - } - - @Test - public void testValidCurrencyUSD() throws ValidatorLoaderException { - - helper.load(tableDescriptor, "amount,currency", "1.50,USD"); - - assertThat(helper.getValidationNotices()).isEmpty(); - } - - @Test - public void testInvalidCurrencyUSD() throws ValidatorLoaderException { - helper.load(tableDescriptor, "amount,currency", "1.5,USD"); - - assertThat(helper.getValidationNotices()) - .containsExactly( - new InvalidCurrencyAmountNotice( - "currency_amount.txt", "amount", 2, new BigDecimal("1.5"))); - } - - @Test - public void testValidCurrencyISK() throws ValidatorLoaderException { - // Icelandic króna expects no digits after decimal separator. - helper.load(tableDescriptor, "amount,currency", "5,ISK"); - - assertThat(helper.getValidationNotices()).isEmpty(); - } - - @Test - public void testInvalidCurrencyISK() throws ValidatorLoaderException { - // Icelandic króna expects no digits after decimal separator. - helper.load(tableDescriptor, "amount,currency", "5.0,ISK"); - - assertThat(helper.getValidationNotices()) - .containsExactly( - new InvalidCurrencyAmountNotice( - "currency_amount.txt", "amount", 2, new BigDecimal("5.0"))); - } -} diff --git a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java deleted file mode 100644 index 46a4b9a1ca..0000000000 --- a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.mobilitydata.gtfsvalidator.processor.tests; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.common.collect.ImmutableList; -import java.util.Arrays; -import java.util.Collection; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.mobilitydata.gtfsvalidator.notice.MixedCaseRecommendedFieldNotice; -import org.mobilitydata.gtfsvalidator.table.MixedCaseTest; -import org.mobilitydata.gtfsvalidator.table.MixedCaseTestTableDescriptor; -import org.mobilitydata.gtfsvalidator.testing.LoadingHelper; -import org.mobilitydata.gtfsvalidator.validator.MixedCaseTestMixedCaseValidator; -import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader; -import org.mobilitydata.gtfsvalidator.validator.ValidatorLoaderException; - -@RunWith(Parameterized.class) -public class MixedCaseSchemaTest { - private final String value; - private final boolean isValid; - - public MixedCaseSchemaTest(String value, boolean isValid) { - this.value = value; - this.isValid = isValid; - } - - private MixedCaseTestTableDescriptor tableDescriptor; - private LoadingHelper helper; - - @Before - public void setup() throws ValidatorLoaderException { - tableDescriptor = new MixedCaseTestTableDescriptor(); - helper = new LoadingHelper(); - helper.setValidatorLoader( - ValidatorLoader.createForClasses(ImmutableList.of(MixedCaseTestMixedCaseValidator.class))); - } - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList( - new Object[][] { - // valid values - {"Mixed-Case", true}, - {"Mixed_Case", true}, - {"Mixed Case", true}, - {"22222", true}, - {"A1", true}, - {"ZA112", true}, - {"301", true}, - {"RTE 30", true}, - {"급 행 12", true}, - {"급행12", true}, - {"東西線", true}, - {"101B", true}, - {"A14C", true}, - {"A14c", true}, - {"A14-C", true}, - {"A14_C", true}, - {"A14 C", true}, - {"Route 1", true}, - {"Route 1 Boulevard", true}, - {"ZA12", true}, - {"Avenue des Champs-Élysées", true}, - {"UPPERCASE", true}, - {"ROUTE 22", true}, - {"ROUTE 1", true}, - {"route 1 Boulevard", true}, - {"Sentences are ok with one mixed case word", true}, - {"MixedCaseButSingleWord", true}, - // invalid values - {"lowercase", false}, - {"snake_case", false}, - {"kebab-case", false}, - {"UPPER-CASE", false}, - {"lower case space", false}, - {"34broadst", false}, - }); - } - - @Test - public void testMixedCase() throws ValidatorLoaderException { - helper.load(tableDescriptor, MixedCaseTest.SOME_FIELD_FIELD_NAME, value); - if (isValid) { - assertThat(helper.getValidationNotices()).isEmpty(); - } else { - assertThat(helper.getValidationNotices()) - .containsExactly( - new MixedCaseRecommendedFieldNotice( - MixedCaseTest.FILENAME, MixedCaseTest.SOME_FIELD_FIELD_NAME, value, 2)); - } - } -} From 204f931aa9f94ba8b47585b039742c20773c7e48 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:42:58 -0400 Subject: [PATCH 02/28] fix FeedMetadata compilation issues --- .../gtfsvalidator/table/GtfsFeedContainer.java | 5 +++-- .../report/model/FeedMetadata.java | 17 +++++++++++------ .../TranslationFieldAndReferenceValidator.java | 17 +++++++++-------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java index d05deb0f7f..142d6b82d9 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java @@ -48,8 +48,9 @@ public GtfsFeedContainer(List> tableContainerList) { * @param filename file name, including ".txt" extension * @return GTFS table or empty if the table is not supported by schema */ - public Optional> getTableForFilename(String filename) { - return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null)); + public > Optional getTableForFilename(String filename) { + return (Optional) + Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null)); } public > T getTable(Class clazz) { diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index fc9c1d6aba..9bd51bf732 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -77,11 +77,15 @@ public static FeedMetadata from(GtfsFeedContainer feedContainer, ImmutableSet> + feedInfoTableOptional = feedContainer.getTableForFilename(GtfsFeedInfo.FILENAME); + feedMetadata.loadFeedInfo(feedInfoTableOptional.get()); + } + if (feedContainer.getTableForFilename(GtfsAgency.FILENAME).isPresent()) { + Optional> agencyTableOptional = + feedContainer.getTableForFilename(GtfsAgency.FILENAME); + feedMetadata.loadAgencyData(agencyTableOptional.get()); } - feedMetadata.loadAgencyData( - (GtfsContainer) feedContainer.getTableForFilename(GtfsAgency.FILENAME).get()); feedMetadata.loadSpecFeatures(feedContainer); return feedMetadata; } @@ -292,13 +296,14 @@ private void loadRouteColorsFeature(GtfsFeedContainer feedContainer) { List.of((Function) GtfsRoute::hasRouteTextColor))); } - private void loadAgencyData(GtfsContainer agencyTable) { + private void loadAgencyData(GtfsContainer agencyTable) { for (GtfsAgency agency : agencyTable.getEntities()) { agencies.add(AgencyMetadata.from(agency)); } } - private void loadFeedInfo(GtfsContainer feedTable) { + private void loadFeedInfo( + GtfsTableContainer feedTable) { var info = feedTable.getEntities().isEmpty() ? null : feedTable.getEntities().get(0); feedInfo.put(FEED_INFO_PUBLISHER_NAME, info == null ? "N/A" : info.feedPublisherName()); diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java index 45dcc2aabd..98ec78baa2 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java @@ -27,11 +27,7 @@ import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFieldNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; -import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTranslation; -import org.mobilitydata.gtfsvalidator.table.GtfsTranslationSchema; -import org.mobilitydata.gtfsvalidator.table.GtfsTranslationTableContainer; +import org.mobilitydata.gtfsvalidator.table.*; /** * Validates that translations are provided in accordance with GTFS Specification. @@ -125,12 +121,17 @@ private void validateTranslation(GtfsTranslation translation, NoticeContainer no translation, GtfsTranslation.RECORD_SUB_ID_FIELD_NAME, translation.recordSubId())); } } - Optional> parentTable = + Optional> parentTable = feedContainer.getTableForFilename(translation.tableName() + ".txt"); if (parentTable.isEmpty() || parentTable.get().isMissingFile()) { noticeContainer.addValidationNotice(new TranslationUnknownTableNameNotice(translation)); } else if (!translation.hasFieldValue()) { - validateReferenceIntegrity(translation, parentTable.get(), noticeContainer); + if (parentTable.isPresent() && parentTable.get() instanceof GtfsTableContainer) { + validateReferenceIntegrity( + translation, (GtfsTableContainer) parentTable.get(), noticeContainer); + } else { + // TODO check for JSON Tables here + } } } @@ -140,7 +141,7 @@ private void validateTranslation(GtfsTranslation translation, NoticeContainer no */ private void validateReferenceIntegrity( GtfsTranslation translation, - GtfsTableContainer parentTable, + GtfsTableContainer parentTable, NoticeContainer noticeContainer) { ImmutableList keyColumnNames = parentTable.getKeyColumnNames(); if (isMissingOrUnexpectedField( From e7751f0241daefe2419a48b50ebf3fbfbeb06fa9 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:44:23 -0400 Subject: [PATCH 03/28] add generated files to gitignore --- .gitignore | 1 + web/client/.gitignore | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index af4b5e8a58..d9f2809763 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ app/pkg/bin/ processor/notices/bin/ processor/notices/tests/bin/ web/service/bin/ +/web/service/execution_result.json diff --git a/web/client/.gitignore b/web/client/.gitignore index fef3daace4..d907181126 100644 --- a/web/client/.gitignore +++ b/web/client/.gitignore @@ -12,3 +12,4 @@ vite.config.ts.timestamp-* rules.json cypress/screenshots/ cypress/videos/ +/static/RULES.md From f027e9a205a1ea07e1b316ea0ddcb38d5626b93f Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:23:15 -0400 Subject: [PATCH 04/28] update TableStatus use --- .../validator/MissingFeedInfoValidatorTest.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java index 0dd0553dff..43b109f93a 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java @@ -25,9 +25,8 @@ private static List generateNotices( public void missingFeedInfoTranslationTableNotPresent() { assertThat( generateNotices( - GtfsFeedInfoTableContainer.forStatus(GtfsTableContainer.TableStatus.MISSING_FILE), - GtfsTranslationTableContainer.forStatus( - GtfsTableContainer.TableStatus.MISSING_FILE))) + GtfsFeedInfoTableContainer.forStatus(TableStatus.MISSING_FILE), + GtfsTranslationTableContainer.forStatus(TableStatus.MISSING_FILE))) .containsExactly(new MissingRecommendedFileNotice(GtfsFeedInfo.FILENAME)); } @@ -35,9 +34,8 @@ public void missingFeedInfoTranslationTableNotPresent() { public void missingFeedInfoWhenTranslationTableIsPresent() { assertThat( generateNotices( - GtfsFeedInfoTableContainer.forStatus(GtfsTableContainer.TableStatus.MISSING_FILE), - GtfsTranslationTableContainer.forStatus( - GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS))) + GtfsFeedInfoTableContainer.forStatus(TableStatus.MISSING_FILE), + GtfsTranslationTableContainer.forStatus(TableStatus.PARSABLE_HEADERS_AND_ROWS))) .contains(new MissingRequiredFileNotice(GtfsFeedInfo.FILENAME)); } @@ -45,10 +43,8 @@ public void missingFeedInfoWhenTranslationTableIsPresent() { public void feedInfoPresentShouldGenerateNoNotice() { assertThat( generateNotices( - GtfsFeedInfoTableContainer.forStatus( - GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS), - GtfsTranslationTableContainer.forStatus( - GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS))) + GtfsFeedInfoTableContainer.forStatus(TableStatus.PARSABLE_HEADERS_AND_ROWS), + GtfsTranslationTableContainer.forStatus(TableStatus.PARSABLE_HEADERS_AND_ROWS))) .isEmpty(); } } From 2eae957ad8e4fc486987200d4eb27a8eac0b630c Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:36:50 -0400 Subject: [PATCH 05/28] add json container class --- .../table/GtfsJsonContainer.java | 9 ++++ .../table/GtfsTableContainer.java | 46 ------------------- 2 files changed, 9 insertions(+), 46 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java new file mode 100644 index 0000000000..d02c2d4dcc --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java @@ -0,0 +1,9 @@ +package org.mobilitydata.gtfsvalidator.table; + +public abstract class GtfsJsonContainer> + extends GtfsContainer { + + public GtfsJsonContainer(D descriptor, TableStatus tableStatus) { + super(descriptor, tableStatus); + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java index e6d1114a92..c94ba0f7fb 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java @@ -34,23 +34,13 @@ public abstract class GtfsTableContainer extends GtfsContainer { - private final D descriptor; - - private final TableStatus tableStatus; - private final CsvHeader header; public GtfsTableContainer(D descriptor, TableStatus tableStatus, CsvHeader header) { super(descriptor, tableStatus); - this.descriptor = descriptor; - this.tableStatus = tableStatus; this.header = header; } - public TableStatus getTableStatus() { - return tableStatus; - } - public CsvHeader getHeader() { return header; } @@ -91,40 +81,4 @@ public boolean hasColumn(String columnName) { * @return entity with the given translation record id, if any */ public abstract Optional byTranslationKey(String recordId, String recordSubId); - - /** - * Tells if the file is missing. - * - * @return true if the file is missing, false otherwise - */ - public boolean isMissingFile() { - return tableStatus == TableStatus.MISSING_FILE; - } - - /** - * Tells if the file was successfully parsed. - * - *

If all files in the feed were successfully parsed, then file validators may be executed. - * - *

A successfully parsed file must meet the following conditions: - * - *

    - *
  • the file was successfully parsed as CSV; - *
  • all headers are valid, required headers are present; - *
  • all rows are successfully parsed; - *
  • if the file is required, it is present in the feed. - *
- * - * @return true if file was successfully parsed, false otherwise - */ - public boolean isParsedSuccessfully() { - switch (tableStatus) { - case PARSABLE_HEADERS_AND_ROWS: - return true; - case MISSING_FILE: - return !descriptor.isRequired(); - default: - return false; - } - } } From ae500ef5b7056e221b7086006af2e5c88ff536c4 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:17:21 -0400 Subject: [PATCH 06/28] prepare feed container to load json files --- .../gtfsvalidator/table/AnyTableLoader.java | 7 +++-- .../gtfsvalidator/table/GtfsDescriptor.java | 4 +++ .../gtfsvalidator/table/GtfsFeedLoader.java | 26 ++++++++++++------- .../table/GtfsTableDescriptor.java | 1 + .../validator/ClassGraphDiscovery.java | 10 +++---- .../validator/DefaultValidatorProvider.java | 2 +- .../validator/ValidatorProvider.java | 2 +- .../report/model/FeedMetadataTest.java | 2 +- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java index 526c539e4a..13a03c75fa 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java @@ -192,13 +192,12 @@ private static void logFieldCacheStats( } } - public static GtfsTableContainer loadMissingFile( - GtfsTableDescriptor tableDescriptor, + public static GtfsContainer loadMissingFile( + GtfsDescriptor tableDescriptor, ValidatorProvider validatorProvider, NoticeContainer noticeContainer) { String gtfsFilename = tableDescriptor.gtfsFilename(); - GtfsTableContainer table = - tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); + GtfsContainer table = tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); if (tableDescriptor.isRecommended()) { noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java index bdcbcaffb5..f1de71d042 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java @@ -2,6 +2,10 @@ // TODO: review class name maybe GtfsFileDescriptor public abstract class GtfsDescriptor { + + public abstract C createContainerForInvalidStatus( + TableStatus tableStatus); + // True if the specified file is required in a feed. private boolean required; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index a4586b6ead..c4b37cd5d7 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -46,7 +46,7 @@ */ public class GtfsFeedLoader { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private final HashMap> tableDescriptors = new HashMap<>(); + private final HashMap> tableDescriptors = new HashMap<>(); private int numThreads = 1; /** @@ -56,12 +56,11 @@ public class GtfsFeedLoader { private final List> multiFileValidatorsWithParsingErrors = new ArrayList<>(); - public GtfsFeedLoader( - ImmutableList>> tableDescriptorClasses) { - for (Class> clazz : tableDescriptorClasses) { - GtfsTableDescriptor descriptor; + public GtfsFeedLoader(ImmutableList>> tableDescriptorClasses) { + for (Class> clazz : tableDescriptorClasses) { + GtfsDescriptor descriptor; try { - descriptor = clazz.asSubclass(GtfsTableDescriptor.class).getConstructor().newInstance(); + descriptor = clazz.asSubclass(GtfsDescriptor.class).getConstructor().newInstance(); } catch (ReflectiveOperationException e) { logger.atSevere().withCause(e).log( "Possible bug in GTFS annotation processor: expected a constructor without parameters" @@ -73,7 +72,7 @@ public GtfsFeedLoader( } } - public Collection> getTableDescriptors() { + public Collection> getTableDescriptors() { return Collections.unmodifiableCollection(tableDescriptors.values()); } @@ -132,9 +131,16 @@ public GtfsFeedContainer loadAndValidate( } ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); - for (GtfsTableDescriptor tableDescriptor : remainingDescriptors.values()) { - tableContainers.add( - AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); + for (GtfsDescriptor tableDescriptor : remainingDescriptors.values()) { + if (tableDescriptor instanceof GtfsTableDescriptor) { + tableContainers.add( + AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); + } else { + // TODO Load JSON file here + logger.atWarning().log( + "Table descriptor %s is not a GtfsTableDescriptor, skipping", + tableDescriptor.getClass().getCanonicalName()); + } } try { for (Future futureContainer : exec.invokeAll(loaderCallables)) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java index e448db4972..74a2f76a57 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java @@ -9,6 +9,7 @@ public abstract class GtfsTableDescriptor extends GtfsDescriptor { + @Override public abstract GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus); public abstract GtfsTableContainer createContainerForHeaderAndEntities( diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java index f435c3e079..170305eedc 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java @@ -7,7 +7,7 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; import org.mobilitydata.gtfsvalidator.notice.Notice; -import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.GtfsDescriptor; /** Discovers GTFS table descriptor and validator classes in the given Java packages. */ public class ClassGraphDiscovery { @@ -23,8 +23,8 @@ private ClassGraphDiscovery() {} /** Discovers GtfsTableDescriptor subclasses in the default table package. */ @SuppressWarnings("unchecked") - public static ImmutableList>> discoverTables() { - ImmutableList.Builder>> tableDescriptors = + public static ImmutableList>> discoverTables() { + ImmutableList.Builder>> tableDescriptors = ImmutableList.builder(); try (ScanResult scanResult = new ClassGraph() @@ -32,8 +32,8 @@ public static ImmutableList>> discoverTab .enableAnnotationInfo() .acceptPackages(DEFAULT_TABLE_PACKAGE) .scan()) { - for (ClassInfo classInfo : scanResult.getSubclasses(GtfsTableDescriptor.class)) { - tableDescriptors.add((Class>) classInfo.loadClass()); + for (ClassInfo classInfo : scanResult.getSubclasses(GtfsDescriptor.class)) { + tableDescriptors.add((Class>) classInfo.loadClass()); } } return tableDescriptors.build(); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java index 3628f844fc..5af7d70904 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java @@ -103,7 +103,7 @@ public List> createSingleEntityV @SuppressWarnings("unchecked") public List createSingleFileValidators( - GtfsTableContainer table, + GtfsContainer table, Consumer> validatorsWithParsingErrors) { List validators = new ArrayList<>(); for (Class validatorClass : diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java index 84ea9880bb..9e60330059 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java @@ -57,7 +57,7 @@ List> createSingleEntityValidato */ List createSingleFileValidators( - GtfsTableContainer table, + GtfsContainer table, Consumer> validatorsWithParsingErrors); /** diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java index cdebe4c4f6..45632b602f 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java @@ -53,7 +53,7 @@ public void setup() throws IOException, ValidatorLoaderException { private void validateSpecFeature( String specFeature, Boolean expectedValue, - ImmutableList>> tableDescriptors) + ImmutableList>> tableDescriptors) throws IOException, InterruptedException { NoticeContainer noticeContainer = new NoticeContainer(); feedLoaderMock = new GtfsFeedLoader(tableDescriptors); From acab7165152bddd536e26acdfdd81637a5579c19 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:44:16 -0400 Subject: [PATCH 07/28] skip abstract classes while initializing the gtfs descriptors --- .../org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index c4b37cd5d7..f5800579ef 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import java.io.InputStream; +import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -60,6 +61,10 @@ public GtfsFeedLoader(ImmutableList>> tableDes for (Class> clazz : tableDescriptorClasses) { GtfsDescriptor descriptor; try { + // Skipping abstract classes. Example: GtfsTableDescriptor. + if (Modifier.isAbstract(clazz.getModifiers())) { + continue; + } descriptor = clazz.asSubclass(GtfsDescriptor.class).getConstructor().newInstance(); } catch (ReflectiveOperationException e) { logger.atSevere().withCause(e).log( From 138d55066b382f9fa2a7966a7152d2e0ca223f4f Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Wed, 28 Aug 2024 19:10:09 -0400 Subject: [PATCH 08/28] add empty GtfsJsonFileLoader --- .../gtfsvalidator/table/GtfsFeedLoader.java | 35 ++++++++++++------- .../table/GtfsJsonDescriptor.java | 10 ++++++ .../gtfsvalidator/table/JsonFileLoader.java | 24 +++++++++++++ 3 files changed, 56 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index f5800579ef..b2050bb3ae 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -104,7 +104,7 @@ public GtfsFeedContainer loadAndValidate( Map> remainingDescriptors = (Map>) tableDescriptors.clone(); for (String filename : gtfsInput.getFilenames()) { - GtfsTableDescriptor tableDescriptor = remainingDescriptors.remove(filename.toLowerCase()); + GtfsDescriptor tableDescriptor = remainingDescriptors.remove(filename.toLowerCase()); if (tableDescriptor == null) { noticeContainer.addValidationNotice(new UnknownFileNotice(filename)); } else { @@ -114,9 +114,26 @@ public GtfsFeedContainer loadAndValidate( GtfsContainer tableContainer; try (InputStream inputStream = gtfsInput.getFile(filename)) { try { - tableContainer = - AnyTableLoader.load( - tableDescriptor, validatorProvider, inputStream, loaderNotices); + if (tableDescriptor instanceof GtfsTableDescriptor) { + tableContainer = + AnyTableLoader.load( + (GtfsTableDescriptor) tableDescriptor, + validatorProvider, + inputStream, + loaderNotices); + } else if (tableDescriptor instanceof GtfsJsonDescriptor) { + tableContainer = + JsonFileLoader.load( + (GtfsJsonDescriptor) tableDescriptor, + validatorProvider, + inputStream, + loaderNotices); + } else { + logger.atSevere().log( + "Runtime exception table descriptor not supported: %s", + tableDescriptor.getClass().getName()); + throw new RuntimeException("Table descriptor is not a supported type"); + } } catch (RuntimeException e) { // This handler should prevent ExecutionException for // this thread. We catch an exception here for storing @@ -137,15 +154,7 @@ public GtfsFeedContainer loadAndValidate( ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); for (GtfsDescriptor tableDescriptor : remainingDescriptors.values()) { - if (tableDescriptor instanceof GtfsTableDescriptor) { - tableContainers.add( - AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); - } else { - // TODO Load JSON file here - logger.atWarning().log( - "Table descriptor %s is not a GtfsTableDescriptor, skipping", - tableDescriptor.getClass().getCanonicalName()); - } + AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer); } try { for (Future futureContainer : exec.invokeAll(loaderCallables)) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java new file mode 100644 index 0000000000..e3cc6b63d8 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java @@ -0,0 +1,10 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; + +public abstract class GtfsJsonDescriptor extends GtfsDescriptor { + + public abstract GtfsJsonContainer createContainerForEntities( + List entities, NoticeContainer noticeContainer); +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java new file mode 100644 index 0000000000..f69bba7c69 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -0,0 +1,24 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; + +public class JsonFileLoader { + + public static GtfsJsonContainer load( + GtfsJsonDescriptor tableDescriptor, + ValidatorProvider validatorProvider, + InputStream inputStream, + NoticeContainer noticeContainer) { + final List entities = new ArrayList<>(); + GtfsJsonContainer table = tableDescriptor.createContainerForEntities(entities, noticeContainer); + // ValidatorUtil.invokeSingleFileValidators( + // validatorProvider.createSingleFileValidators( + // table, singleFileValidatorsWithParsingErrors::add), + // noticeContainer); + return table; + } +} From e1ea3f114e745ae721a04d2ce07bb29ecbbb9d53 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 29 Aug 2024 15:52:27 -0400 Subject: [PATCH 09/28] Corrected a NPE caused by containers missing for files not dataset --- .../org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index b2050bb3ae..9223b387ac 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -154,7 +154,7 @@ public GtfsFeedContainer loadAndValidate( ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); for (GtfsDescriptor tableDescriptor : remainingDescriptors.values()) { - AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer); + tableContainers.add(AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); } try { for (Future futureContainer : exec.invokeAll(loaderCallables)) { From 435876e50503119f80376357e19067ccf4fc2c99 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 3 Sep 2024 23:31:38 -0400 Subject: [PATCH 10/28] We now have a working POC with a notice if feature id is present in stops.txt --- .../UniqueLocationIdViolationNotice.java | 61 ++++++++++++ .../gtfsvalidator/table/GtfsFeedLoader.java | 3 +- .../gtfsvalidator/table/GtfsJson.java | 88 +++++++++++++++++ .../table/GtfsJsonContainer.java | 65 +++++++++++- .../table/GtfsJsonDescriptor.java | 30 +++++- .../gtfsvalidator/table/JsonFileLoader.java | 99 +++++++++++++++++-- .../GtfsJsonUniqueLocationIdValidator.java | 86 ++++++++++++++++ 7 files changed, 418 insertions(+), 14 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java new file mode 100644 index 0000000000..dc317897f4 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mobilitydata.gtfsvalidator.notice; + +import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS; +import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.UrlRef; + +/** + * Location id from locations.geojson already exists. + * + *

The id of one of the features of the locations.geojson file already exists in stops.txt or + * location_groups.txt + */ +@GtfsValidationNotice( + severity = ERROR, + sections = @SectionRefs(FILE_REQUIREMENTS), + urls = { + @UrlRef( + label = "Original Python validator implementation", + url = "https://github.com/google/transitfeed") + }) +public class UniqueLocationIdViolationNotice extends ValidationNotice { + + /** The id that already exists. */ + private final String id; + + /** The name of the file that already has this id. */ + private final String fileWithIdAlreadyPresent; + + /** The name of the field that contains this id. */ + private final String fieldNameInFile; + + /** The row of the record in the file where the id is already present. */ + private final int csvRowNumber; + + public UniqueLocationIdViolationNotice( + String id, String fileWithIdAlreadyPresent, String fieldNameInFile, int csvRowNumber) { + + this.id = id; + this.fileWithIdAlreadyPresent = fileWithIdAlreadyPresent; + this.fieldNameInFile = fieldNameInFile; + this.csvRowNumber = csvRowNumber; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 9223b387ac..76f84501f7 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -154,7 +154,8 @@ public GtfsFeedContainer loadAndValidate( ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); for (GtfsDescriptor tableDescriptor : remainingDescriptors.values()) { - tableContainers.add(AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); + tableContainers.add( + AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); } try { for (Future futureContainer : exec.invokeAll(loaderCallables)) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java new file mode 100644 index 0000000000..02edfe7479 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java @@ -0,0 +1,88 @@ +package org.mobilitydata.gtfsvalidator.table; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public final class GtfsJson implements GtfsEntity { + public static final String FILENAME = "locations.geojson"; + + public static final String LOCATION_ID_FIELD_NAME = "location_id"; + + private String locationId; // The id of a feature in the GeoJSON file. + + public GtfsJson() {} + + @Override + public int csvRowNumber() { + return 0; + } + + @Nonnull + public String locationId() { + return locationId; + } + + public boolean hasLocationId() { + return locationId != null; + } + + public void setLocationId(@Nullable String locationId) { + this.locationId = locationId; + } + + // public static final class Builder implements GtfsEntityBuilder { + // private static final String DEFAULT_LOCATION_ID = null; + // private int csvRowNumber; + // + // private String locationId; + // + // public Builder() { + // // Initialize all fields to default values. + // clear(); + // } + // + // @Override + // public int csvRowNumber() { + // return csvRowNumber; + // } + // + // @Override + // public GtfsJson.Builder setCsvRowNumber(int value) { + // csvRowNumber = value; + // return this; + // } + // + // @Nonnull + // public String locationId() { + // return locationId; + // } + // + // @Nonnull + // public GtfsJson.Builder setLocationId(@Nullable String value) { + // if (value == null) { + // return clearLocationId(); + // } + // locationId = value; + // return this; + // } + // + // @Nonnull + // public GtfsJson.Builder clearLocationId() { + // locationId = DEFAULT_LOCATION_ID; + // return this; + // } + // + // @Override + // public GtfsJson build() { + // GtfsJson entity = new GtfsJson(); + // + // return entity; + // } + // + // @Override + // public void clear() { + // csvRowNumber = 0; + // locationId = DEFAULT_LOCATION_ID; + // } + // } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java index d02c2d4dcc..532109305f 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java @@ -1,9 +1,68 @@ package org.mobilitydata.gtfsvalidator.table; -public abstract class GtfsJsonContainer> +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.mobilitydata.gtfsvalidator.notice.DuplicateKeyNotice; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; + +public class GtfsJsonContainer extends GtfsContainer { - public GtfsJsonContainer(D descriptor, TableStatus tableStatus) { - super(descriptor, tableStatus); + private final Map byLocationIdMap = new HashMap<>(); + + private final List entities; + + public GtfsJsonContainer(D descriptor, List entities, NoticeContainer noticeContainer) { + super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS); + this.entities = entities; + setupIndices(noticeContainer); + } + + public GtfsJsonContainer(GtfsDescriptor descriptor, TableStatus tableStatus) { + super((D) descriptor, tableStatus); + this.entities = new ArrayList<>(); + } + + @Override + public Class getEntityClass() { + return (Class) GtfsJson.class; + } + + @Override + public List getEntities() { + return (List) entities; + } + + @Override + public String gtfsFilename() { + return "locations.geojson"; + } + + @Override + public Optional byTranslationKey(String recordId, String recordSubId) { + return Optional.empty(); + } + + private void setupIndices(NoticeContainer noticeContainer) { + for (GtfsJson newEntity : entities) { + if (!newEntity.hasLocationId()) { + continue; + } + GtfsJson oldEntity = byLocationIdMap.getOrDefault(newEntity.locationId(), null); + if (oldEntity != null) { + noticeContainer.addValidationNotice( + new DuplicateKeyNotice( + gtfsFilename(), + newEntity.csvRowNumber(), + oldEntity.csvRowNumber(), + GtfsJson.LOCATION_ID_FIELD_NAME, + newEntity.locationId())); + } else { + byLocationIdMap.put(newEntity.locationId(), newEntity); + } + } } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java index e3cc6b63d8..b812438172 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java @@ -3,8 +3,32 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public abstract class GtfsJsonDescriptor extends GtfsDescriptor { +public class GtfsJsonDescriptor extends GtfsDescriptor +// GtfsDescriptor +{ - public abstract GtfsJsonContainer createContainerForEntities( - List entities, NoticeContainer noticeContainer); + public GtfsJsonContainer createContainerForEntities( + List entities, NoticeContainer noticeContainer) { + return new GtfsJsonContainer(this, entities, noticeContainer); + } + + @Override + public C createContainerForInvalidStatus(TableStatus tableStatus) { + return (C) new GtfsJsonContainer(this, tableStatus); + } + + @Override + public boolean isRecommended() { + return false; + } + + @Override + public Class getEntityClass() { + return GtfsJson.class; + } + + @Override + public String gtfsFilename() { + return "locations.geojson"; + } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java index f69bba7c69..9d96748d9e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -1,8 +1,15 @@ package org.mobilitydata.gtfsvalidator.table; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.IOError; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; @@ -13,12 +20,90 @@ public static GtfsJsonContainer load( ValidatorProvider validatorProvider, InputStream inputStream, NoticeContainer noticeContainer) { - final List entities = new ArrayList<>(); - GtfsJsonContainer table = tableDescriptor.createContainerForEntities(entities, noticeContainer); - // ValidatorUtil.invokeSingleFileValidators( - // validatorProvider.createSingleFileValidators( - // table, singleFileValidatorsWithParsingErrors::add), - // noticeContainer); - return table; + try { + List entities = extractFeaturesFromStream(inputStream, noticeContainer); +// List locationIds = extractIdsFromStream(inputStream); + + +// for (String locationId : locationIds) { +// GtfsJson entity = new GtfsJson(); +// entity.setLocationId(locationId); +// // builder.setLocationId(locationId); +// entities.add(entity); +// } + + GtfsJsonContainer container = + tableDescriptor.createContainerForEntities(entities, noticeContainer); + return container; + } catch (IOException ioex) { + noticeContainer.addSystemError(new IOError(ioex)); + return (GtfsJsonContainer) + tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); + } + } + + public static List extractFeaturesFromStream(InputStream inputStream, NoticeContainer noticeContainer) throws IOException { + List features = new ArrayList<>(); + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); + JsonArray featuresArray = jsonObject.getAsJsonArray("features"); + for (JsonElement feature : featuresArray) { + GtfsJson gtfsJson = extractFeature(feature, noticeContainer); + if (gtfsJson != null) { + features.add(gtfsJson); + } + } + } + return features; + } + + public static GtfsJson extractFeature(JsonElement feature, NoticeContainer noticeContainer) { + GtfsJson gtfsJson = new GtfsJson(); + if (feature.isJsonObject()) { + JsonObject featureObject = feature.getAsJsonObject(); + if (featureObject.has("properties")) { + JsonObject properties = featureObject.getAsJsonObject("properties"); + // Add stop_name and stop_desc + } else { + // Add a notice because properties is required + } + if (featureObject.has("id")) { + gtfsJson.setLocationId(featureObject.get("id").getAsString()); + } else { + // Add a notice because id is required + } + + if (featureObject.has("geometry")) { + JsonObject geometry = featureObject.getAsJsonObject("geometry"); + if (geometry.has("type")) { + String type = geometry.get("type").getAsString(); + if (type.equals("Polygon")) { + // Extract the polygon + } else if (type.equals("Multipolygon")) { + // extract the multipolygon + } + } else { + // Add a notice because type is required + } + } else { + // Add a notice because geometry is required + } + } + return gtfsJson; + } + + public static List extractIdsFromStream(InputStream inputStream) throws IOException { + List ids = new ArrayList<>(); + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); + JsonArray features = jsonObject.getAsJsonArray("features"); + for (JsonElement feature : features) { + JsonObject featureObject = feature.getAsJsonObject(); + if (featureObject.has("id")) { + ids.add(featureObject.get("id").getAsString()); + } + } + } + return ids; } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java new file mode 100644 index 0000000000..2d61bc77f3 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.validator; + +import java.util.Optional; +import javax.inject.Inject; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.notice.UniqueLocationIdViolationNotice; +import org.mobilitydata.gtfsvalidator.table.GtfsJson; +import org.mobilitydata.gtfsvalidator.table.GtfsJsonContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsJsonDescriptor; +import org.mobilitydata.gtfsvalidator.table.GtfsNetworkTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsRouteTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsStop; +import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; + +/** + * Validates that the location id from "locations.geojson" is not a duplicate of any stop_id from + * "stops.txt" or location_group_id from "location_gorups.txt" + * + *

Generated notice: {@link UniqueLocationIdViolationNotice}. + */ +@GtfsValidator +public class GtfsJsonUniqueLocationIdValidator extends FileValidator { + private final GtfsStopTableContainer stopTableContainer; + + // Remove comments when the location_group_stops.txt file is added to the GTFS schema + // private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; + + private final GtfsJsonContainer> jsonContainer; + + @Inject + GtfsJsonUniqueLocationIdValidator( + GtfsJsonContainer> jsonContainer, + GtfsStopTableContainer stopTableContainer + // , GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer + ) { + this.jsonContainer = jsonContainer; + + this.stopTableContainer = stopTableContainer; + // this.locationGroupStopsTableContainer = locationGroupStopsTableContainer; + } + + @Override + public void validate(NoticeContainer noticeContainer) { + for (GtfsJson json : jsonContainer.getEntities()) { + String locationId = json.locationId(); + if (locationId.isEmpty()) { + continue; + } + + Optional stop = stopTableContainer.byStopId(locationId); + if (stop.isPresent()) { + noticeContainer.addValidationNotice( + new UniqueLocationIdViolationNotice( + locationId, + GtfsStop.FILENAME, + GtfsStop.STOP_ID_FIELD_NAME, + stop.get().csvRowNumber())); + } + } + } + + private boolean hasReferencedKey( + String foreignKey, + GtfsRouteTableContainer routeContainer, + GtfsNetworkTableContainer networkContainer) { + return !(routeContainer.byNetworkId(foreignKey).isEmpty() + && networkContainer.byNetworkId(foreignKey).isEmpty()); + } +} From 75e8d80eae31f431b0ed521d7827ed9613f91477 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 4 Sep 2024 00:04:00 -0400 Subject: [PATCH 11/28] Changed names of some classes --- .../UniqueLocationIdViolationNotice.java | 2 +- .../gtfsvalidator/table/GtfsFeedLoader.java | 4 +- ...{GtfsJson.java => GtfsGeojsonFeature.java} | 6 +-- ...java => GtfsGeojsonFeatureDescriptor.java} | 13 +++--- ...java => GtfsGeojsonFeaturesContainer.java} | 18 ++++---- .../gtfsvalidator/table/JsonFileLoader.java | 45 ++++++++++--------- ...jsonFeatureUniqueLocationIdValidator.java} | 32 ++++++------- 7 files changed, 60 insertions(+), 60 deletions(-) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{GtfsJson.java => GtfsGeojsonFeature.java} (91%) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{GtfsJsonDescriptor.java => GtfsGeojsonFeatureDescriptor.java} (55%) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{GtfsJsonContainer.java => GtfsGeojsonFeaturesContainer.java} (68%) rename main/src/main/java/org/mobilitydata/gtfsvalidator/validator/{GtfsJsonUniqueLocationIdValidator.java => GtfsGeojsonFeatureUniqueLocationIdValidator.java} (69%) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java index dc317897f4..cbd308c950 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java @@ -23,7 +23,7 @@ import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.UrlRef; /** - * Location id from locations.geojson already exists. + * Feature id from locations.geojson already used. * *

The id of one of the features of the locations.geojson file already exists in stops.txt or * location_groups.txt diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 76f84501f7..f69787ad7e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -121,10 +121,10 @@ public GtfsFeedContainer loadAndValidate( validatorProvider, inputStream, loaderNotices); - } else if (tableDescriptor instanceof GtfsJsonDescriptor) { + } else if (tableDescriptor instanceof GtfsGeojsonFeatureDescriptor) { tableContainer = JsonFileLoader.load( - (GtfsJsonDescriptor) tableDescriptor, + (GtfsGeojsonFeatureDescriptor) tableDescriptor, validatorProvider, inputStream, loaderNotices); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java similarity index 91% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java index 02edfe7479..d378cc57ce 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJson.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java @@ -3,14 +3,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -public final class GtfsJson implements GtfsEntity { +public final class GtfsGeojsonFeature implements GtfsEntity { public static final String FILENAME = "locations.geojson"; public static final String LOCATION_ID_FIELD_NAME = "location_id"; - private String locationId; // The id of a feature in the GeoJSON file. + private String locationId; // The id of a feature in the GeoJSON file. - public GtfsJson() {} + public GtfsGeojsonFeature() {} @Override public int csvRowNumber() { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureDescriptor.java similarity index 55% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureDescriptor.java index b812438172..800baba20b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureDescriptor.java @@ -3,18 +3,19 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public class GtfsJsonDescriptor extends GtfsDescriptor +public class GtfsGeojsonFeatureDescriptor + extends GtfsDescriptor // GtfsDescriptor { - public GtfsJsonContainer createContainerForEntities( + public GtfsGeojsonFeaturesContainer createContainerForEntities( List entities, NoticeContainer noticeContainer) { - return new GtfsJsonContainer(this, entities, noticeContainer); + return new GtfsGeojsonFeaturesContainer(this, entities, noticeContainer); } @Override public C createContainerForInvalidStatus(TableStatus tableStatus) { - return (C) new GtfsJsonContainer(this, tableStatus); + return (C) new GtfsGeojsonFeaturesContainer(this, tableStatus); } @Override @@ -23,8 +24,8 @@ public boolean isRecommended() { } @Override - public Class getEntityClass() { - return GtfsJson.class; + public Class getEntityClass() { + return GtfsGeojsonFeature.class; } @Override diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java similarity index 68% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index 532109305f..f41d7cdc64 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -8,27 +8,29 @@ import org.mobilitydata.gtfsvalidator.notice.DuplicateKeyNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public class GtfsJsonContainer +public class GtfsGeojsonFeaturesContainer extends GtfsContainer { - private final Map byLocationIdMap = new HashMap<>(); + private final Map byLocationIdMap = new HashMap<>(); private final List entities; - public GtfsJsonContainer(D descriptor, List entities, NoticeContainer noticeContainer) { + public GtfsGeojsonFeaturesContainer( + D descriptor, List entities, NoticeContainer noticeContainer) { super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS); this.entities = entities; setupIndices(noticeContainer); } - public GtfsJsonContainer(GtfsDescriptor descriptor, TableStatus tableStatus) { + public GtfsGeojsonFeaturesContainer( + GtfsDescriptor descriptor, TableStatus tableStatus) { super((D) descriptor, tableStatus); this.entities = new ArrayList<>(); } @Override public Class getEntityClass() { - return (Class) GtfsJson.class; + return (Class) GtfsGeojsonFeature.class; } @Override @@ -47,18 +49,18 @@ public Optional byTranslationKey(String recordId, String recordSubId) { } private void setupIndices(NoticeContainer noticeContainer) { - for (GtfsJson newEntity : entities) { + for (GtfsGeojsonFeature newEntity : entities) { if (!newEntity.hasLocationId()) { continue; } - GtfsJson oldEntity = byLocationIdMap.getOrDefault(newEntity.locationId(), null); + GtfsGeojsonFeature oldEntity = byLocationIdMap.getOrDefault(newEntity.locationId(), null); if (oldEntity != null) { noticeContainer.addValidationNotice( new DuplicateKeyNotice( gtfsFilename(), newEntity.csvRowNumber(), oldEntity.csvRowNumber(), - GtfsJson.LOCATION_ID_FIELD_NAME, + GtfsGeojsonFeature.LOCATION_ID_FIELD_NAME, newEntity.locationId())); } else { byLocationIdMap.put(newEntity.locationId(), newEntity); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java index 9d96748d9e..989e67486b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -15,50 +15,51 @@ public class JsonFileLoader { - public static GtfsJsonContainer load( - GtfsJsonDescriptor tableDescriptor, + public static GtfsGeojsonFeaturesContainer load( + GtfsGeojsonFeatureDescriptor tableDescriptor, ValidatorProvider validatorProvider, InputStream inputStream, NoticeContainer noticeContainer) { try { - List entities = extractFeaturesFromStream(inputStream, noticeContainer); -// List locationIds = extractIdsFromStream(inputStream); + List entities = extractFeaturesFromStream(inputStream, noticeContainer); + // List locationIds = extractIdsFromStream(inputStream); + // for (String locationId : locationIds) { + // GtfsJson entity = new GtfsJson(); + // entity.setLocationId(locationId); + // // builder.setLocationId(locationId); + // entities.add(entity); + // } -// for (String locationId : locationIds) { -// GtfsJson entity = new GtfsJson(); -// entity.setLocationId(locationId); -// // builder.setLocationId(locationId); -// entities.add(entity); -// } - - GtfsJsonContainer container = + GtfsGeojsonFeaturesContainer container = tableDescriptor.createContainerForEntities(entities, noticeContainer); return container; } catch (IOException ioex) { noticeContainer.addSystemError(new IOError(ioex)); - return (GtfsJsonContainer) + return (GtfsGeojsonFeaturesContainer) tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); } } - public static List extractFeaturesFromStream(InputStream inputStream, NoticeContainer noticeContainer) throws IOException { - List features = new ArrayList<>(); + public static List extractFeaturesFromStream( + InputStream inputStream, NoticeContainer noticeContainer) throws IOException { + List features = new ArrayList<>(); try (InputStreamReader reader = new InputStreamReader(inputStream)) { JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); JsonArray featuresArray = jsonObject.getAsJsonArray("features"); for (JsonElement feature : featuresArray) { - GtfsJson gtfsJson = extractFeature(feature, noticeContainer); - if (gtfsJson != null) { - features.add(gtfsJson); + GtfsGeojsonFeature gtfsGeojsonFeature = extractFeature(feature, noticeContainer); + if (gtfsGeojsonFeature != null) { + features.add(gtfsGeojsonFeature); } } } return features; } - public static GtfsJson extractFeature(JsonElement feature, NoticeContainer noticeContainer) { - GtfsJson gtfsJson = new GtfsJson(); + public static GtfsGeojsonFeature extractFeature( + JsonElement feature, NoticeContainer noticeContainer) { + GtfsGeojsonFeature gtfsGeojsonFeature = new GtfsGeojsonFeature(); if (feature.isJsonObject()) { JsonObject featureObject = feature.getAsJsonObject(); if (featureObject.has("properties")) { @@ -68,7 +69,7 @@ public static GtfsJson extractFeature(JsonElement feature, NoticeContainer notic // Add a notice because properties is required } if (featureObject.has("id")) { - gtfsJson.setLocationId(featureObject.get("id").getAsString()); + gtfsGeojsonFeature.setLocationId(featureObject.get("id").getAsString()); } else { // Add a notice because id is required } @@ -89,7 +90,7 @@ public static GtfsJson extractFeature(JsonElement feature, NoticeContainer notic // Add a notice because geometry is required } } - return gtfsJson; + return gtfsGeojsonFeature; } public static List extractIdsFromStream(InputStream inputStream) throws IOException { diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java similarity index 69% rename from main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java index 2d61bc77f3..30e189ba2d 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsJsonUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java @@ -21,36 +21,39 @@ import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.UniqueLocationIdViolationNotice; -import org.mobilitydata.gtfsvalidator.table.GtfsJson; -import org.mobilitydata.gtfsvalidator.table.GtfsJsonContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsJsonDescriptor; +import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; +import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeatureDescriptor; import org.mobilitydata.gtfsvalidator.table.GtfsNetworkTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsRouteTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsStop; import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; /** - * Validates that the location id from "locations.geojson" is not a duplicate of any stop_id from - * "stops.txt" or location_group_id from "location_gorups.txt" + * Validates that the feature id from "locations.geojson" is not a duplicate of any stop_id from + * "stops.txt" or location_group_id from "location_groups.txt" * *

Generated notice: {@link UniqueLocationIdViolationNotice}. */ @GtfsValidator -public class GtfsJsonUniqueLocationIdValidator extends FileValidator { +public class GtfsGeojsonFeatureUniqueLocationIdValidator extends FileValidator { private final GtfsStopTableContainer stopTableContainer; // Remove comments when the location_group_stops.txt file is added to the GTFS schema // private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; - private final GtfsJsonContainer> jsonContainer; + private final GtfsGeojsonFeaturesContainer> + geojsonFeatureContainer; @Inject - GtfsJsonUniqueLocationIdValidator( - GtfsJsonContainer> jsonContainer, + GtfsGeojsonFeatureUniqueLocationIdValidator( + GtfsGeojsonFeaturesContainer< + GtfsGeojsonFeature, GtfsGeojsonFeatureDescriptor> + geojsonFeatureContainer, GtfsStopTableContainer stopTableContainer // , GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer ) { - this.jsonContainer = jsonContainer; + this.geojsonFeatureContainer = geojsonFeatureContainer; this.stopTableContainer = stopTableContainer; // this.locationGroupStopsTableContainer = locationGroupStopsTableContainer; @@ -58,7 +61,7 @@ public class GtfsJsonUniqueLocationIdValidator extends FileValidator { @Override public void validate(NoticeContainer noticeContainer) { - for (GtfsJson json : jsonContainer.getEntities()) { + for (GtfsGeojsonFeature json : geojsonFeatureContainer.getEntities()) { String locationId = json.locationId(); if (locationId.isEmpty()) { continue; @@ -76,11 +79,4 @@ public void validate(NoticeContainer noticeContainer) { } } - private boolean hasReferencedKey( - String foreignKey, - GtfsRouteTableContainer routeContainer, - GtfsNetworkTableContainer networkContainer) { - return !(routeContainer.byNetworkId(foreignKey).isEmpty() - && networkContainer.byNetworkId(foreignKey).isEmpty()); - } } From 1e9fac57876e5362949c90127decfec3eb324990 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 4 Sep 2024 09:27:19 -0400 Subject: [PATCH 12/28] Reformatting --- ...sGeojsonFeatureUniqueLocationIdValidator.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java index 30e189ba2d..adb3d8f620 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java @@ -22,10 +22,8 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.UniqueLocationIdViolationNotice; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; -import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeatureDescriptor; -import org.mobilitydata.gtfsvalidator.table.GtfsNetworkTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsRouteTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; import org.mobilitydata.gtfsvalidator.table.GtfsStop; import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; @@ -39,17 +37,18 @@ public class GtfsGeojsonFeatureUniqueLocationIdValidator extends FileValidator { private final GtfsStopTableContainer stopTableContainer; - // Remove comments when the location_group_stops.txt file is added to the GTFS schema + // Remove this comment when the location_group_stops.txt file is added to the GTFS schema // private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; - private final GtfsGeojsonFeaturesContainer> - geojsonFeatureContainer; + private final GtfsGeojsonFeaturesContainer< + GtfsGeojsonFeature, GtfsGeojsonFeatureDescriptor> + geojsonFeatureContainer; @Inject GtfsGeojsonFeatureUniqueLocationIdValidator( GtfsGeojsonFeaturesContainer< - GtfsGeojsonFeature, GtfsGeojsonFeatureDescriptor> - geojsonFeatureContainer, + GtfsGeojsonFeature, GtfsGeojsonFeatureDescriptor> + geojsonFeatureContainer, GtfsStopTableContainer stopTableContainer // , GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer ) { @@ -78,5 +77,4 @@ public void validate(NoticeContainer noticeContainer) { } } } - } From 209d203fc68fe0fcd5a2cddec97fb742f0c5069d Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 4 Sep 2024 11:00:47 -0400 Subject: [PATCH 13/28] Added new notices fields. --- .../gtfsvalidator/validator/NoticeFieldsTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java index 6ed212c10e..36c14c2a50 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java @@ -195,7 +195,10 @@ public void testNoticeClassFieldNames() { "maxShapeDistanceTraveled", "maxTripDistanceTraveled", "fileNameA", - "fileNameB"); + "fileNameB", + "id", + "fileWithIdAlreadyPresent", + "fieldNameInFile"); } private static List discoverValidationNoticeFieldNames() { From b82b39b12dbdea6e48cdbf9da533cdac9dfa1326 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 4 Sep 2024 15:50:15 -0400 Subject: [PATCH 14/28] Added class comments and renamed some classes. --- .../gtfsvalidator/table/AnyTableLoader.java | 7 +++-- ...ontainer.java => GtfsEntityContainer.java} | 12 ++++++-- .../table/GtfsFeedContainer.java | 18 +++++------ .../gtfsvalidator/table/GtfsFeedLoader.java | 30 ++++++++++--------- ...escriptor.java => GtfsFileDescriptor.java} | 14 ++++++--- .../table/GtfsGeojsonFeaturesContainer.java | 7 +++-- ...or.java => GtfsGeojsonFileDescriptor.java} | 7 +++-- .../table/GtfsTableContainer.java | 2 +- .../table/GtfsTableDescriptor.java | 2 +- .../gtfsvalidator/table/JsonFileLoader.java | 27 ++--------------- .../validator/ClassGraphDiscovery.java | 10 +++---- .../validator/DefaultValidatorProvider.java | 5 ++-- .../validator/ValidatorLoader.java | 28 +++++++++-------- .../validator/ValidatorProvider.java | 2 +- .../report/model/FeedMetadata.java | 7 +++-- .../report/model/TableMetadata.java | 4 +-- ...ojsonFeatureUniqueLocationIdValidator.java | 6 ++-- ...TranslationFieldAndReferenceValidator.java | 2 +- .../report/model/FeedMetadataTest.java | 2 +- 19 files changed, 96 insertions(+), 96 deletions(-) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{GtfsContainer.java => GtfsEntityContainer.java} (64%) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{GtfsDescriptor.java => GtfsFileDescriptor.java} (52%) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{GtfsGeojsonFeatureDescriptor.java => GtfsGeojsonFileDescriptor.java} (76%) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java index 13a03c75fa..e176261c29 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java @@ -192,12 +192,13 @@ private static void logFieldCacheStats( } } - public static GtfsContainer loadMissingFile( - GtfsDescriptor tableDescriptor, + public static GtfsEntityContainer loadMissingFile( + GtfsFileDescriptor tableDescriptor, ValidatorProvider validatorProvider, NoticeContainer noticeContainer) { String gtfsFilename = tableDescriptor.gtfsFilename(); - GtfsContainer table = tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); + GtfsEntityContainer table = + tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); if (tableDescriptor.isRecommended()) { noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java similarity index 64% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java index c5ab7ce385..1e7bf96d32 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java @@ -3,12 +3,20 @@ import java.util.List; import java.util.Optional; -public abstract class GtfsContainer { +/** + * This class is the parent of the containers holding table (csv) entities and containers holding + * geojson entities + * + * @param The entity for this container (e.g. GtfsCalendarDate or {@link GtfsGeojsonFeature} ) + * @param The descriptor for the table for the container (e.g. GtfsCalendarDateTableDescriptor + * or {@link GtfsGeojsonFileDescriptor}) + */ +public abstract class GtfsEntityContainer { private final D descriptor; private final TableStatus tableStatus; - public GtfsContainer(D descriptor, TableStatus tableStatus) { + public GtfsEntityContainer(D descriptor, TableStatus tableStatus) { this.tableStatus = tableStatus; this.descriptor = descriptor; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java index 142d6b82d9..6731df5a49 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java @@ -25,12 +25,12 @@ *

The tables are kept as {@code GtfsContainer} instances. */ public class GtfsFeedContainer { - private final Map> tables = new HashMap<>(); - private final Map, GtfsContainer> tablesByClass = + private final Map> tables = new HashMap<>(); + private final Map, GtfsEntityContainer> tablesByClass = new HashMap<>(); - public GtfsFeedContainer(List> tableContainerList) { - for (GtfsContainer table : tableContainerList) { + public GtfsFeedContainer(List> tableContainerList) { + for (GtfsEntityContainer table : tableContainerList) { tables.put(table.gtfsFilename(), table); tablesByClass.put(table.getClass(), table); } @@ -48,12 +48,12 @@ public GtfsFeedContainer(List> tableContainerList) { * @param filename file name, including ".txt" extension * @return GTFS table or empty if the table is not supported by schema */ - public > Optional getTableForFilename(String filename) { + public > Optional getTableForFilename(String filename) { return (Optional) Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null)); } - public > T getTable(Class clazz) { + public > T getTable(Class clazz) { return (T) tablesByClass.get(clazz); } @@ -65,7 +65,7 @@ public GtfsFeedContainer(List> tableContainerList) { * @return true if all files were successfully parsed, false otherwise */ public boolean isParsedSuccessfully() { - for (GtfsContainer table : tables.values()) { + for (GtfsEntityContainer table : tables.values()) { if (!table.isParsedSuccessfully()) { return false; } @@ -73,13 +73,13 @@ public boolean isParsedSuccessfully() { return true; } - public Collection> getTables() { + public Collection> getTables() { return tables.values(); } public String tableTotalsText() { List totalList = new ArrayList<>(); - for (GtfsContainer table : tables.values()) { + for (GtfsEntityContainer table : tables.values()) { if (table.getTableStatus() == TableStatus.MISSING_FILE && !table.getDescriptor().isRequired()) { continue; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index f69787ad7e..769aa23e5b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -47,7 +47,7 @@ */ public class GtfsFeedLoader { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private final HashMap> tableDescriptors = new HashMap<>(); + private final HashMap> tableDescriptors = new HashMap<>(); private int numThreads = 1; /** @@ -57,15 +57,16 @@ public class GtfsFeedLoader { private final List> multiFileValidatorsWithParsingErrors = new ArrayList<>(); - public GtfsFeedLoader(ImmutableList>> tableDescriptorClasses) { - for (Class> clazz : tableDescriptorClasses) { - GtfsDescriptor descriptor; + public GtfsFeedLoader( + ImmutableList>> tableDescriptorClasses) { + for (Class> clazz : tableDescriptorClasses) { + GtfsFileDescriptor descriptor; try { // Skipping abstract classes. Example: GtfsTableDescriptor. if (Modifier.isAbstract(clazz.getModifiers())) { continue; } - descriptor = clazz.asSubclass(GtfsDescriptor.class).getConstructor().newInstance(); + descriptor = clazz.asSubclass(GtfsFileDescriptor.class).getConstructor().newInstance(); } catch (ReflectiveOperationException e) { logger.atSevere().withCause(e).log( "Possible bug in GTFS annotation processor: expected a constructor without parameters" @@ -77,7 +78,7 @@ public GtfsFeedLoader(ImmutableList>> tableDes } } - public Collection> getTableDescriptors() { + public Collection> getTableDescriptors() { return Collections.unmodifiableCollection(tableDescriptors.values()); } @@ -104,14 +105,14 @@ public GtfsFeedContainer loadAndValidate( Map> remainingDescriptors = (Map>) tableDescriptors.clone(); for (String filename : gtfsInput.getFilenames()) { - GtfsDescriptor tableDescriptor = remainingDescriptors.remove(filename.toLowerCase()); + GtfsFileDescriptor tableDescriptor = remainingDescriptors.remove(filename.toLowerCase()); if (tableDescriptor == null) { noticeContainer.addValidationNotice(new UnknownFileNotice(filename)); } else { loaderCallables.add( () -> { NoticeContainer loaderNotices = new NoticeContainer(); - GtfsContainer tableContainer; + GtfsEntityContainer tableContainer; try (InputStream inputStream = gtfsInput.getFile(filename)) { try { if (tableDescriptor instanceof GtfsTableDescriptor) { @@ -121,10 +122,10 @@ public GtfsFeedContainer loadAndValidate( validatorProvider, inputStream, loaderNotices); - } else if (tableDescriptor instanceof GtfsGeojsonFeatureDescriptor) { + } else if (tableDescriptor instanceof GtfsGeojsonFileDescriptor) { tableContainer = JsonFileLoader.load( - (GtfsGeojsonFeatureDescriptor) tableDescriptor, + (GtfsGeojsonFileDescriptor) tableDescriptor, validatorProvider, inputStream, loaderNotices); @@ -151,9 +152,9 @@ public GtfsFeedContainer loadAndValidate( }); } } - ArrayList> tableContainers = new ArrayList<>(); + ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); - for (GtfsDescriptor tableDescriptor : remainingDescriptors.values()) { + for (GtfsFileDescriptor tableDescriptor : remainingDescriptors.values()) { tableContainers.add( AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); } @@ -207,10 +208,11 @@ private static void addThreadExecutionError( } static class TableAndNoticeContainers { - final GtfsContainer tableContainer; + final GtfsEntityContainer tableContainer; final NoticeContainer noticeContainer; - public TableAndNoticeContainers(GtfsContainer tableContainer, NoticeContainer noticeContainer) { + public TableAndNoticeContainers( + GtfsEntityContainer tableContainer, NoticeContainer noticeContainer) { this.tableContainer = tableContainer; this.noticeContainer = noticeContainer; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java similarity index 52% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java index f1de71d042..479351167e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java @@ -1,9 +1,15 @@ package org.mobilitydata.gtfsvalidator.table; -// TODO: review class name maybe GtfsFileDescriptor -public abstract class GtfsDescriptor { - - public abstract C createContainerForInvalidStatus( +/** + * This class provides soem info about the different files within a GTFS dataset. Its children + * relate to either a csv table or a geojson file. + * + * @param The entity that will be extracted from the file. For example, GtfsCalendarDate or + * {@link GtfsGeojsonFeature} + */ +public abstract class GtfsFileDescriptor { + + public abstract C createContainerForInvalidStatus( TableStatus tableStatus); // True if the specified file is required in a feed. diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index f41d7cdc64..8438de639b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -8,8 +8,9 @@ import org.mobilitydata.gtfsvalidator.notice.DuplicateKeyNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public class GtfsGeojsonFeaturesContainer - extends GtfsContainer { +public class GtfsGeojsonFeaturesContainer< + T extends GtfsGeojsonFeature, D extends GtfsFileDescriptor> + extends GtfsEntityContainer { private final Map byLocationIdMap = new HashMap<>(); @@ -23,7 +24,7 @@ public GtfsGeojsonFeaturesContainer( } public GtfsGeojsonFeaturesContainer( - GtfsDescriptor descriptor, TableStatus tableStatus) { + GtfsFileDescriptor descriptor, TableStatus tableStatus) { super((D) descriptor, tableStatus); this.entities = new ArrayList<>(); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java similarity index 76% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureDescriptor.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index 800baba20b..f1d6571dc9 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -3,8 +3,8 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public class GtfsGeojsonFeatureDescriptor - extends GtfsDescriptor +public class GtfsGeojsonFileDescriptor + extends GtfsFileDescriptor // GtfsDescriptor { @@ -14,7 +14,8 @@ public GtfsGeojsonFeaturesContainer createContainerForEntities( } @Override - public C createContainerForInvalidStatus(TableStatus tableStatus) { + public C createContainerForInvalidStatus( + TableStatus tableStatus) { return (C) new GtfsGeojsonFeaturesContainer(this, tableStatus); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java index c94ba0f7fb..0ea8258f80 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java @@ -32,7 +32,7 @@ * @param subclass of {@code GtfsEntity} */ public abstract class GtfsTableContainer - extends GtfsContainer { + extends GtfsEntityContainer { private final CsvHeader header; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java index 74a2f76a57..b7a6bf7600 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java @@ -7,7 +7,7 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; -public abstract class GtfsTableDescriptor extends GtfsDescriptor { +public abstract class GtfsTableDescriptor extends GtfsFileDescriptor { @Override public abstract GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java index 989e67486b..de6dd7cc01 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -16,22 +16,14 @@ public class JsonFileLoader { public static GtfsGeojsonFeaturesContainer load( - GtfsGeojsonFeatureDescriptor tableDescriptor, + GtfsGeojsonFileDescriptor tableDescriptor, ValidatorProvider validatorProvider, InputStream inputStream, NoticeContainer noticeContainer) { try { List entities = extractFeaturesFromStream(inputStream, noticeContainer); - // List locationIds = extractIdsFromStream(inputStream); - // for (String locationId : locationIds) { - // GtfsJson entity = new GtfsJson(); - // entity.setLocationId(locationId); - // // builder.setLocationId(locationId); - // entities.add(entity); - // } - - GtfsGeojsonFeaturesContainer container = + GtfsGeojsonFeaturesContainer container = tableDescriptor.createContainerForEntities(entities, noticeContainer); return container; } catch (IOException ioex) { @@ -92,19 +84,4 @@ public static GtfsGeojsonFeature extractFeature( } return gtfsGeojsonFeature; } - - public static List extractIdsFromStream(InputStream inputStream) throws IOException { - List ids = new ArrayList<>(); - try (InputStreamReader reader = new InputStreamReader(inputStream)) { - JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); - JsonArray features = jsonObject.getAsJsonArray("features"); - for (JsonElement feature : features) { - JsonObject featureObject = feature.getAsJsonObject(); - if (featureObject.has("id")) { - ids.add(featureObject.get("id").getAsString()); - } - } - } - return ids; - } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java index 170305eedc..4d0073377c 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java @@ -7,7 +7,7 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; import org.mobilitydata.gtfsvalidator.notice.Notice; -import org.mobilitydata.gtfsvalidator.table.GtfsDescriptor; +import org.mobilitydata.gtfsvalidator.table.GtfsFileDescriptor; /** Discovers GTFS table descriptor and validator classes in the given Java packages. */ public class ClassGraphDiscovery { @@ -23,8 +23,8 @@ private ClassGraphDiscovery() {} /** Discovers GtfsTableDescriptor subclasses in the default table package. */ @SuppressWarnings("unchecked") - public static ImmutableList>> discoverTables() { - ImmutableList.Builder>> tableDescriptors = + public static ImmutableList>> discoverTables() { + ImmutableList.Builder>> tableDescriptors = ImmutableList.builder(); try (ScanResult scanResult = new ClassGraph() @@ -32,8 +32,8 @@ public static ImmutableList>> discoverTables() .enableAnnotationInfo() .acceptPackages(DEFAULT_TABLE_PACKAGE) .scan()) { - for (ClassInfo classInfo : scanResult.getSubclasses(GtfsDescriptor.class)) { - tableDescriptors.add((Class>) classInfo.loadClass()); + for (ClassInfo classInfo : scanResult.getSubclasses(GtfsFileDescriptor.class)) { + tableDescriptors.add((Class>) classInfo.loadClass()); } } return tableDescriptors.build(); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java index 5af7d70904..32f55e6583 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java @@ -33,7 +33,8 @@ public class DefaultValidatorProvider implements ValidatorProvider { private final TableHeaderValidator tableHeaderValidator; private final ListMultimap, Class>> singleEntityValidators; - private final ListMultimap>, Class> + private final ListMultimap< + Class>, Class> singleFileValidators; private final List> multiFileValidators; @@ -103,7 +104,7 @@ public List> createSingleEntityV @SuppressWarnings("unchecked") public List createSingleFileValidators( - GtfsContainer table, + GtfsEntityContainer table, Consumer> validatorsWithParsingErrors) { List validators = new ArrayList<>(); for (Class validatorClass : diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java index 43e089d569..bb4e6b3c8a 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java @@ -29,8 +29,8 @@ import javax.annotation.Nullable; import javax.inject.Inject; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsContainer; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; /** @@ -43,7 +43,8 @@ public class ValidatorLoader { private final ListMultimap, Class>> singleEntityValidators = ArrayListMultimap.create(); - private final ListMultimap>, Class> + private final ListMultimap< + Class>, Class> singleFileValidators = ArrayListMultimap.create(); private final List> multiFileValidators = new ArrayList<>(); @@ -75,7 +76,7 @@ private ValidatorLoader() {} } /** Loaded single-file validator classes keyed by table container class. */ - public ListMultimap>, Class> + public ListMultimap>, Class> getSingleFileValidators() { return singleFileValidators; } @@ -113,14 +114,14 @@ private void addFileValidator(Class validatorClass) // Indicates that the full GtfsFeedContainer needs to be injected. boolean injectFeedContainer = false; // Find out which GTFS tables need to be injected. - List>> injectedTables = new ArrayList<>(); + List>> injectedTables = new ArrayList<>(); for (Class parameterType : constructor.getParameterTypes()) { if (GtfsFeedContainer.class.isAssignableFrom(parameterType)) { injectFeedContainer = true; continue; } - if (GtfsContainer.class.isAssignableFrom(parameterType)) { - injectedTables.add((Class>) parameterType); + if (GtfsEntityContainer.class.isAssignableFrom(parameterType)) { + injectedTables.add((Class>) parameterType); } } @@ -201,7 +202,7 @@ ValidatorWithDependencyStatus createValidatorWithContext( public static ValidatorWithDependencyStatus createSingleFileValidator( Class clazz, - GtfsContainer table, + GtfsEntityContainer table, ValidationContext validationContext) throws ReflectiveOperationException, ValidatorLoaderException { return (ValidatorWithDependencyStatus) @@ -222,7 +223,7 @@ public static ValidatorWithDependencyStatus createM */ private static class DependencyResolver { private final ValidationContext context; - @Nullable private final GtfsContainer tableContainer; + @Nullable private final GtfsEntityContainer tableContainer; @Nullable private final GtfsFeedContainer feedContainer; /** This will be set to true if a resolved dependency was not parsed successfully. */ @@ -230,7 +231,7 @@ private static class DependencyResolver { public DependencyResolver( ValidationContext context, - @Nullable GtfsContainer tableContainer, + @Nullable GtfsEntityContainer tableContainer, @Nullable GtfsFeedContainer feedContainer) { this.context = context; this.tableContainer = tableContainer; @@ -257,9 +258,9 @@ public Object resolveDependency(Class parameterClass) { } return tableContainer; } - if (feedContainer != null && GtfsContainer.class.isAssignableFrom(parameterClass)) { - GtfsContainer container = - feedContainer.getTable((Class>) parameterClass); + if (feedContainer != null && GtfsEntityContainer.class.isAssignableFrom(parameterClass)) { + GtfsEntityContainer container = + feedContainer.getTable((Class>) parameterClass); if (container != null && !container.isParsedSuccessfully()) { dependenciesHaveErrors = true; } @@ -305,7 +306,8 @@ public String listValidators() { if (!singleFileValidators.isEmpty()) { builder.append("Single-file validators\n"); for (Map.Entry< - Class>, Collection>> + Class>, + Collection>> entry : singleFileValidators.asMap().entrySet()) { builder.append("\t").append(entry.getKey().getSimpleName()).append(": "); for (Class validatorClass : entry.getValue()) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java index 9e60330059..cab1124841 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java @@ -57,7 +57,7 @@ List> createSingleEntityValidato */ List createSingleFileValidators( - GtfsContainer table, + GtfsEntityContainer table, Consumer> validatorsWithParsingErrors); /** diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index 9bd51bf732..381243c898 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -104,7 +104,7 @@ private void setCounts(GtfsFeedContainer feedContainer) { setCount(COUNTS_BLOCKS, feedContainer, GtfsTrip.FILENAME, GtfsTrip.class, GtfsTrip::blockId); } - private void setCount( + private void setCount( String countName, GtfsFeedContainer feedContainer, String fileName, @@ -118,7 +118,7 @@ private void setCount( } private int loadUniqueCount( - GtfsContainer table, Class clazz, Function idExtractor) { + GtfsEntityContainer table, Class clazz, Function idExtractor) { // Iterate through entities and count unique IDs Set uniqueIds = new HashSet<>(); for (GtfsEntity entity : table.getEntities()) { @@ -296,7 +296,8 @@ private void loadRouteColorsFeature(GtfsFeedContainer feedContainer) { List.of((Function) GtfsRoute::hasRouteTextColor))); } - private void loadAgencyData(GtfsContainer agencyTable) { + private void loadAgencyData( + GtfsEntityContainer agencyTable) { for (GtfsAgency agency : agencyTable.getEntities()) { agencies.add(AgencyMetadata.from(agency)); } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java index e1c1dc7ba4..cd34b9400b 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java @@ -1,6 +1,6 @@ package org.mobilitydata.gtfsvalidator.report.model; -import org.mobilitydata.gtfsvalidator.table.GtfsContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; import org.mobilitydata.gtfsvalidator.table.TableStatus; public class TableMetadata { @@ -15,7 +15,7 @@ public TableMetadata(String filename, TableStatus tableStatus, int entityCount) this.entityCount = entityCount; } - public static TableMetadata from(GtfsContainer table) { + public static TableMetadata from(GtfsEntityContainer table) { return new TableMetadata(table.gtfsFilename(), table.getTableStatus(), table.entityCount()); } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java index adb3d8f620..8449090c4a 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java @@ -22,8 +22,8 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.UniqueLocationIdViolationNotice; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; -import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeatureDescriptor; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFileDescriptor; import org.mobilitydata.gtfsvalidator.table.GtfsStop; import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; @@ -41,13 +41,13 @@ public class GtfsGeojsonFeatureUniqueLocationIdValidator extends FileValidator { // private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; private final GtfsGeojsonFeaturesContainer< - GtfsGeojsonFeature, GtfsGeojsonFeatureDescriptor> + GtfsGeojsonFeature, GtfsGeojsonFileDescriptor> geojsonFeatureContainer; @Inject GtfsGeojsonFeatureUniqueLocationIdValidator( GtfsGeojsonFeaturesContainer< - GtfsGeojsonFeature, GtfsGeojsonFeatureDescriptor> + GtfsGeojsonFeature, GtfsGeojsonFileDescriptor> geojsonFeatureContainer, GtfsStopTableContainer stopTableContainer // , GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java index 98ec78baa2..2073577482 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java @@ -121,7 +121,7 @@ private void validateTranslation(GtfsTranslation translation, NoticeContainer no translation, GtfsTranslation.RECORD_SUB_ID_FIELD_NAME, translation.recordSubId())); } } - Optional> parentTable = + Optional> parentTable = feedContainer.getTableForFilename(translation.tableName() + ".txt"); if (parentTable.isEmpty() || parentTable.get().isMissingFile()) { noticeContainer.addValidationNotice(new TranslationUnknownTableNameNotice(translation)); diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java index 45632b602f..4221b9b192 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java @@ -53,7 +53,7 @@ public void setup() throws IOException, ValidatorLoaderException { private void validateSpecFeature( String specFeature, Boolean expectedValue, - ImmutableList>> tableDescriptors) + ImmutableList>> tableDescriptors) throws IOException, InterruptedException { NoticeContainer noticeContainer = new NoticeContainer(); feedLoaderMock = new GtfsFeedLoader(tableDescriptors); From 8bdab6d1f4f8f02becbdda9489cae0192c7973e4 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 4 Sep 2024 17:54:14 -0400 Subject: [PATCH 15/28] Removed some commented out code --- .../table/GtfsGeojsonFeature.java | 56 ------------------- .../table/GtfsGeojsonFileDescriptor.java | 4 +- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java index d378cc57ce..187da30894 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java @@ -29,60 +29,4 @@ public boolean hasLocationId() { public void setLocationId(@Nullable String locationId) { this.locationId = locationId; } - - // public static final class Builder implements GtfsEntityBuilder { - // private static final String DEFAULT_LOCATION_ID = null; - // private int csvRowNumber; - // - // private String locationId; - // - // public Builder() { - // // Initialize all fields to default values. - // clear(); - // } - // - // @Override - // public int csvRowNumber() { - // return csvRowNumber; - // } - // - // @Override - // public GtfsJson.Builder setCsvRowNumber(int value) { - // csvRowNumber = value; - // return this; - // } - // - // @Nonnull - // public String locationId() { - // return locationId; - // } - // - // @Nonnull - // public GtfsJson.Builder setLocationId(@Nullable String value) { - // if (value == null) { - // return clearLocationId(); - // } - // locationId = value; - // return this; - // } - // - // @Nonnull - // public GtfsJson.Builder clearLocationId() { - // locationId = DEFAULT_LOCATION_ID; - // return this; - // } - // - // @Override - // public GtfsJson build() { - // GtfsJson entity = new GtfsJson(); - // - // return entity; - // } - // - // @Override - // public void clear() { - // csvRowNumber = 0; - // locationId = DEFAULT_LOCATION_ID; - // } - // } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index f1d6571dc9..8e4213811a 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -4,9 +4,7 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; public class GtfsGeojsonFileDescriptor - extends GtfsFileDescriptor -// GtfsDescriptor -{ + extends GtfsFileDescriptor { public GtfsGeojsonFeaturesContainer createContainerForEntities( List entities, NoticeContainer noticeContainer) { From 6f82f0368c95322d89939a62b680d4d207b9552d Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 4 Sep 2024 22:44:58 -0400 Subject: [PATCH 16/28] Simplified the use of generics got geojson classes --- .../UniqueLocationIdViolationNotice.java | 10 +------- .../table/GtfsGeojsonFeaturesContainer.java | 25 ++++++++++--------- .../table/GtfsGeojsonFileDescriptor.java | 10 +++----- .../gtfsvalidator/table/JsonFileLoader.java | 13 ++++------ ...ojsonFeatureUniqueLocationIdValidator.java | 25 ++++++++----------- 5 files changed, 33 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java index cbd308c950..eab5c20a9d 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java @@ -20,7 +20,6 @@ import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.UrlRef; /** * Feature id from locations.geojson already used. @@ -28,14 +27,7 @@ *

The id of one of the features of the locations.geojson file already exists in stops.txt or * location_groups.txt */ -@GtfsValidationNotice( - severity = ERROR, - sections = @SectionRefs(FILE_REQUIREMENTS), - urls = { - @UrlRef( - label = "Original Python validator implementation", - url = "https://github.com/google/transitfeed") - }) +@GtfsValidationNotice(severity = ERROR, sections = @SectionRefs(FILE_REQUIREMENTS)) public class UniqueLocationIdViolationNotice extends ValidationNotice { /** The id that already exists. */ diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index 8438de639b..a87e9802c6 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -8,35 +8,36 @@ import org.mobilitydata.gtfsvalidator.notice.DuplicateKeyNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public class GtfsGeojsonFeaturesContainer< - T extends GtfsGeojsonFeature, D extends GtfsFileDescriptor> - extends GtfsEntityContainer { +public class GtfsGeojsonFeaturesContainer + extends GtfsEntityContainer { private final Map byLocationIdMap = new HashMap<>(); - private final List entities; + private final List entities; public GtfsGeojsonFeaturesContainer( - D descriptor, List entities, NoticeContainer noticeContainer) { + GtfsGeojsonFileDescriptor descriptor, + List entities, + NoticeContainer noticeContainer) { super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS); this.entities = entities; setupIndices(noticeContainer); } public GtfsGeojsonFeaturesContainer( - GtfsFileDescriptor descriptor, TableStatus tableStatus) { - super((D) descriptor, tableStatus); + GtfsGeojsonFileDescriptor descriptor, TableStatus tableStatus) { + super(descriptor, tableStatus); this.entities = new ArrayList<>(); } @Override - public Class getEntityClass() { - return (Class) GtfsGeojsonFeature.class; + public Class getEntityClass() { + return GtfsGeojsonFeature.class; } @Override - public List getEntities() { - return (List) entities; + public List getEntities() { + return entities; } @Override @@ -45,7 +46,7 @@ public String gtfsFilename() { } @Override - public Optional byTranslationKey(String recordId, String recordSubId) { + public Optional byTranslationKey(String recordId, String recordSubId) { return Optional.empty(); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index 8e4213811a..b8ca7005b6 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -3,18 +3,16 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -public class GtfsGeojsonFileDescriptor - extends GtfsFileDescriptor { +public class GtfsGeojsonFileDescriptor extends GtfsFileDescriptor { public GtfsGeojsonFeaturesContainer createContainerForEntities( - List entities, NoticeContainer noticeContainer) { + List entities, NoticeContainer noticeContainer) { return new GtfsGeojsonFeaturesContainer(this, entities, noticeContainer); } @Override - public C createContainerForInvalidStatus( - TableStatus tableStatus) { - return (C) new GtfsGeojsonFeaturesContainer(this, tableStatus); + public GtfsGeojsonFeaturesContainer createContainerForInvalidStatus(TableStatus tableStatus) { + return new GtfsGeojsonFeaturesContainer(this, tableStatus); } @Override diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java index de6dd7cc01..c39816b0b1 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -16,20 +16,16 @@ public class JsonFileLoader { public static GtfsGeojsonFeaturesContainer load( - GtfsGeojsonFileDescriptor tableDescriptor, + GtfsGeojsonFileDescriptor fileDescriptor, ValidatorProvider validatorProvider, InputStream inputStream, NoticeContainer noticeContainer) { try { List entities = extractFeaturesFromStream(inputStream, noticeContainer); - - GtfsGeojsonFeaturesContainer container = - tableDescriptor.createContainerForEntities(entities, noticeContainer); - return container; + return fileDescriptor.createContainerForEntities(entities, noticeContainer); } catch (IOException ioex) { noticeContainer.addSystemError(new IOError(ioex)); - return (GtfsGeojsonFeaturesContainer) - tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); + return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); } } @@ -51,7 +47,7 @@ public static List extractFeaturesFromStream( public static GtfsGeojsonFeature extractFeature( JsonElement feature, NoticeContainer noticeContainer) { - GtfsGeojsonFeature gtfsGeojsonFeature = new GtfsGeojsonFeature(); + GtfsGeojsonFeature gtfsGeojsonFeature = null; if (feature.isJsonObject()) { JsonObject featureObject = feature.getAsJsonObject(); if (featureObject.has("properties")) { @@ -61,6 +57,7 @@ public static GtfsGeojsonFeature extractFeature( // Add a notice because properties is required } if (featureObject.has("id")) { + gtfsGeojsonFeature = new GtfsGeojsonFeature(); gtfsGeojsonFeature.setLocationId(featureObject.get("id").getAsString()); } else { // Add a notice because id is required diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java index 8449090c4a..5bbacdfe6d 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java @@ -23,7 +23,6 @@ import org.mobilitydata.gtfsvalidator.notice.UniqueLocationIdViolationNotice; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFileDescriptor; import org.mobilitydata.gtfsvalidator.table.GtfsStop; import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; @@ -40,15 +39,11 @@ public class GtfsGeojsonFeatureUniqueLocationIdValidator extends FileValidator { // Remove this comment when the location_group_stops.txt file is added to the GTFS schema // private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; - private final GtfsGeojsonFeaturesContainer< - GtfsGeojsonFeature, GtfsGeojsonFileDescriptor> - geojsonFeatureContainer; + private final GtfsGeojsonFeaturesContainer geojsonFeatureContainer; @Inject GtfsGeojsonFeatureUniqueLocationIdValidator( - GtfsGeojsonFeaturesContainer< - GtfsGeojsonFeature, GtfsGeojsonFileDescriptor> - geojsonFeatureContainer, + GtfsGeojsonFeaturesContainer geojsonFeatureContainer, GtfsStopTableContainer stopTableContainer // , GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer ) { @@ -67,14 +62,14 @@ public void validate(NoticeContainer noticeContainer) { } Optional stop = stopTableContainer.byStopId(locationId); - if (stop.isPresent()) { - noticeContainer.addValidationNotice( - new UniqueLocationIdViolationNotice( - locationId, - GtfsStop.FILENAME, - GtfsStop.STOP_ID_FIELD_NAME, - stop.get().csvRowNumber())); - } + stop.ifPresent( + gtfsStop -> + noticeContainer.addValidationNotice( + new UniqueLocationIdViolationNotice( + locationId, + GtfsStop.FILENAME, + GtfsStop.STOP_ID_FIELD_NAME, + gtfsStop.csvRowNumber()))); } } } From 77805f3840e070c969f3af44139985cf0f9d5c60 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 13 Sep 2024 09:42:21 -0400 Subject: [PATCH 17/28] Moved code form core to main. --- .../gtfsvalidator/table/AnyTableLoader.java | 62 +++++----------- .../gtfsvalidator/table/GtfsFeedLoader.java | 24 +++---- .../table/GtfsFileDescriptor.java | 8 ++- .../gtfsvalidator/table/TableLoader.java | 71 +++++++++++++++++++ .../gtfsvalidator/testing/LoadingHelper.java | 2 +- .../table/AnyTableLoaderTest.java | 18 +++-- .../runner/ValidationRunner.java | 14 ++-- .../table/GtfsGeojsonFeature.java | 4 ++ .../table/GtfsGeojsonFeaturesContainer.java | 0 .../table/GtfsGeojsonFileDescriptor.java | 4 ++ .../gtfsvalidator/table/JsonFileLoader.java | 11 +-- 11 files changed, 137 insertions(+), 81 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java rename {core => main}/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java (80%) rename {core => main}/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java (100%) rename {core => main}/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java (91%) rename {core => main}/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java (90%) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java index e176261c29..daae71d285 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java @@ -7,48 +7,42 @@ import com.univocity.parsers.csv.CsvParserSettings; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.mobilitydata.gtfsvalidator.notice.CsvParsingFailedNotice; import org.mobilitydata.gtfsvalidator.notice.EmptyFileNotice; -import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice; -import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvFile; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.parsing.CsvRow; import org.mobilitydata.gtfsvalidator.parsing.FieldCache; import org.mobilitydata.gtfsvalidator.parsing.RowParser; -import org.mobilitydata.gtfsvalidator.validator.FileValidator; import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; -public final class AnyTableLoader { +/** This class loads csv files specifically. */ +public final class AnyTableLoader extends TableLoader { - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final List> singleFileValidatorsWithParsingErrors = - new ArrayList<>(); + private AnyTableLoader() {} + // Create the singleton and add a method to obtain it + private static final AnyTableLoader INSTANCE = new AnyTableLoader(); - private static final List> - singleEntityValidatorsWithParsingErrors = new ArrayList<>(); - - public List> getValidatorsWithParsingErrors() { - return Collections.unmodifiableList(singleFileValidatorsWithParsingErrors); + public static AnyTableLoader getInstance() { + return INSTANCE; } - public List> getSingleEntityValidatorsWithParsingErrors() { - return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors); - } + private final FluentLogger logger = FluentLogger.forEnclosingClass(); - public static GtfsTableContainer load( - GtfsTableDescriptor tableDescriptor, + @Override + public GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, ValidatorProvider validatorProvider, InputStream csvInputStream, NoticeContainer noticeContainer) { + GtfsTableDescriptor tableDescriptor = (GtfsTableDescriptor) fileDescriptor; final String gtfsFilename = tableDescriptor.gtfsFilename(); CsvFile csvFile; @@ -96,8 +90,8 @@ public static GtfsTableContainer load( final List entities = new ArrayList<>(); boolean hasUnparsableRows = false; final List> singleEntityValidators = - validatorProvider.createSingleEntityValidators( - tableDescriptor.getEntityClass(), singleEntityValidatorsWithParsingErrors::add); + createSingleEntityValidators(tableDescriptor.getEntityClass(), validatorProvider); + try { for (CsvRow row : csvFile) { if (row.getRowNumber() % 200000 == 0) { @@ -140,14 +134,13 @@ public static GtfsTableContainer load( } GtfsTableContainer table = tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer); + ValidatorUtil.invokeSingleFileValidators( - validatorProvider.createSingleFileValidators( - table, singleFileValidatorsWithParsingErrors::add), - noticeContainer); + createSingleFileValidators(table, validatorProvider), noticeContainer); return table; } - private static NoticeContainer validateHeaders( + private NoticeContainer validateHeaders( ValidatorProvider validatorProvider, String gtfsFilename, CsvHeader header, @@ -173,7 +166,7 @@ private static NoticeContainer validateHeaders( return headerNotices; } - private static void logFieldCacheStats( + private void logFieldCacheStats( String gtfsFilename, FieldCache[] fieldCaches, ImmutableList columnDescriptors) { @@ -192,23 +185,4 @@ private static void logFieldCacheStats( } } - public static GtfsEntityContainer loadMissingFile( - GtfsFileDescriptor tableDescriptor, - ValidatorProvider validatorProvider, - NoticeContainer noticeContainer) { - String gtfsFilename = tableDescriptor.gtfsFilename(); - GtfsEntityContainer table = - tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); - if (tableDescriptor.isRecommended()) { - noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); - } - if (tableDescriptor.isRequired()) { - noticeContainer.addValidationNotice(new MissingRequiredFileNotice(gtfsFilename)); - } - ValidatorUtil.invokeSingleFileValidators( - validatorProvider.createSingleFileValidators( - table, singleFileValidatorsWithParsingErrors::add), - noticeContainer); - return table; - } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 769aa23e5b..999f10af7c 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -113,22 +113,14 @@ public GtfsFeedContainer loadAndValidate( () -> { NoticeContainer loaderNotices = new NoticeContainer(); GtfsEntityContainer tableContainer; + // The descriptor knows what loader to use to load the file + TableLoader tableLoader = tableDescriptor.getTableLoader(); try (InputStream inputStream = gtfsInput.getFile(filename)) { try { - if (tableDescriptor instanceof GtfsTableDescriptor) { + if (tableLoader != null) { tableContainer = - AnyTableLoader.load( - (GtfsTableDescriptor) tableDescriptor, - validatorProvider, - inputStream, - loaderNotices); - } else if (tableDescriptor instanceof GtfsGeojsonFileDescriptor) { - tableContainer = - JsonFileLoader.load( - (GtfsGeojsonFileDescriptor) tableDescriptor, - validatorProvider, - inputStream, - loaderNotices); + tableLoader.load( + tableDescriptor, validatorProvider, inputStream, loaderNotices); } else { logger.atSevere().log( "Runtime exception table descriptor not supported: %s", @@ -143,8 +135,9 @@ public GtfsFeedContainer loadAndValidate( loaderNotices.addSystemError(new RuntimeExceptionInLoaderError(filename, e)); // Since the file was not loaded successfully, we treat // it as missing for continuing validation. + // tableLoader = tableDescriptor.getTableLoader(); tableContainer = - AnyTableLoader.loadMissingFile( + tableLoader.loadMissingFile( tableDescriptor, validatorProvider, loaderNotices); } } @@ -155,8 +148,9 @@ public GtfsFeedContainer loadAndValidate( ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); for (GtfsFileDescriptor tableDescriptor : remainingDescriptors.values()) { + TableLoader tableLoader = tableDescriptor.getTableLoader(); tableContainers.add( - AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); + tableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); } try { for (Future futureContainer : exec.invokeAll(loaderCallables)) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java index 479351167e..8b06fa40ee 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java @@ -1,11 +1,11 @@ package org.mobilitydata.gtfsvalidator.table; /** - * This class provides soem info about the different files within a GTFS dataset. Its children + * This class provides some info about the different files within a GTFS dataset. Its children * relate to either a csv table or a geojson file. * * @param The entity that will be extracted from the file. For example, GtfsCalendarDate or - * {@link GtfsGeojsonFeature} + * GtfsGeojsonFeature */ public abstract class GtfsFileDescriptor { @@ -30,4 +30,8 @@ public boolean isRequired() { public void setRequired(boolean required) { this.required = required; } + + public TableLoader getTableLoader() { + return AnyTableLoader.getInstance(); + } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java new file mode 100644 index 0000000000..7eb9aecfbf --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java @@ -0,0 +1,71 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice; +import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.validator.FileValidator; +import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator; +import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; +import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; + +public abstract class TableLoader { + + private static final List> + singleEntityValidatorsWithParsingErrors = new ArrayList<>(); + + private static final List> singleFileValidatorsWithParsingErrors = + new ArrayList<>(); + + public static List> getValidatorsWithParsingErrors() { + return Collections.unmodifiableList(singleFileValidatorsWithParsingErrors); + } + + public static List> + getSingleEntityValidatorsWithParsingErrors() { + return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors); + } + + abstract GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, + ValidatorProvider validatorProvider, + InputStream csvInputStream, + NoticeContainer noticeContainer); + + protected List> createSingleEntityValidators( + Class entityClass, ValidatorProvider validatorProvider) { + return validatorProvider.createSingleEntityValidators( + entityClass, singleEntityValidatorsWithParsingErrors::add); + } + + protected + List createSingleFileValidators( + GtfsEntityContainer table, ValidatorProvider validatorProvider) { + + return validatorProvider.createSingleFileValidators( + table, singleFileValidatorsWithParsingErrors::add); + } + + public GtfsEntityContainer loadMissingFile( + GtfsFileDescriptor tableDescriptor, + ValidatorProvider validatorProvider, + NoticeContainer noticeContainer) { + String gtfsFilename = tableDescriptor.gtfsFilename(); + GtfsEntityContainer table = + tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); + if (tableDescriptor.isRecommended()) { + noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); + } + if (tableDescriptor.isRequired()) { + noticeContainer.addValidationNotice(new MissingRequiredFileNotice(gtfsFilename)); + } + ValidatorUtil.invokeSingleFileValidators( + createSingleFileValidators(table, validatorProvider), noticeContainer); + + return table; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java index 1fefbfaa39..f89557fb9a 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java @@ -70,6 +70,6 @@ public void setValidatorLoader(ValidatorLoader validatorLoader) { .setDateForValidation(new DateForValidation(dateForValidation)) .build(); ValidatorProvider provider = new DefaultValidatorProvider(context, validatorLoader); - return (Y) AnyTableLoader.load(tableDescriptor, provider, in, noticeContainer); + return (Y) AnyTableLoader.getInstance().load(tableDescriptor, provider, in, noticeContainer); } } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java index 5dee4d9aca..1c9d63a5d8 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java @@ -50,7 +50,8 @@ public void invalidInputStream() { .thenReturn(mockContainer); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, null, loaderNotices); + AnyTableLoader.getInstance() + .load(testTableDescriptor, validatorProvider, null, loaderNotices); assertThat(validationNoticeTypes(loaderNotices)).containsExactly(CsvParsingFailedNotice.class); assertThat(loadedContainer).isEqualTo(mockContainer); @@ -65,7 +66,8 @@ public void emptyInputStream() { InputStream csvInputStream = toInputStream(""); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); + AnyTableLoader.getInstance() + .load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) .containsExactly(new EmptyFileNotice("filename")); @@ -97,7 +99,8 @@ public void validate( when(validatorProvider.getTableHeaderValidator()).thenReturn(tableHeaderValidator); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); + AnyTableLoader.getInstance() + .load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()).containsExactly(headerValidationNotice); assertThat(loadedContainer).isEqualTo(mockContainer); @@ -112,7 +115,8 @@ public void invalidRowLengthNotice() { InputStream inputStream = toInputStream("id,code\n" + "s1\n"); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); + AnyTableLoader.getInstance() + .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) .containsExactly(new InvalidRowLengthNotice("filename.txt", 2, 1, 2)); @@ -132,7 +136,8 @@ public void parsableTableRows() { InputStream inputStream = toInputStream("id,stop_lat,_no_name_\n" + "s1, 23.00, no_value\n"); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); + AnyTableLoader.getInstance() + .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loadedContainer.getTableStatus()).isEqualTo(TableStatus.PARSABLE_HEADERS_AND_ROWS); verify(validator, times(1)).validate(any()); @@ -167,7 +172,8 @@ public void missingRequiredField() { InputStream inputStream = toInputStream("id,code\n" + "s1,\n"); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); + AnyTableLoader.getInstance() + .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) .contains(new MissingRequiredFieldNotice("filename.txt", 2, "code")); diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java index 1127a25533..214e7d7526 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java @@ -40,9 +40,9 @@ import org.mobilitydata.gtfsvalidator.report.JsonReport; import org.mobilitydata.gtfsvalidator.report.JsonReportGenerator; import org.mobilitydata.gtfsvalidator.report.model.FeedMetadata; -import org.mobilitydata.gtfsvalidator.table.AnyTableLoader; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedLoader; +import org.mobilitydata.gtfsvalidator.table.TableLoader; import org.mobilitydata.gtfsvalidator.util.VersionInfo; import org.mobilitydata.gtfsvalidator.util.VersionResolver; import org.mobilitydata.gtfsvalidator.validator.*; @@ -90,7 +90,6 @@ public Status run(ValidationRunnerConfig config) { return Status.EXCEPTION; } GtfsFeedLoader feedLoader = new GtfsFeedLoader(ClassGraphDiscovery.discoverTables()); - AnyTableLoader anyTableLoader = new AnyTableLoader(); logger.atInfo().log("validation config:\n%s", config); logger.atInfo().log("validators:\n%s", validatorLoader.listValidators()); @@ -138,7 +137,7 @@ public Status run(ValidationRunnerConfig config) { // Output exportReport(feedMetadata, noticeContainer, config, versionInfo); - printSummary(feedMetadata, feedContainer, feedLoader, anyTableLoader); + printSummary(feedMetadata, feedContainer, feedLoader); return Status.SUCCESS; } @@ -149,19 +148,16 @@ public Status run(ValidationRunnerConfig config) { * @param feedContainer the {@code GtfsFeedContainer} */ public static void printSummary( - FeedMetadata feedMetadata, - GtfsFeedContainer feedContainer, - GtfsFeedLoader loader, - AnyTableLoader anyTableLoader) { + FeedMetadata feedMetadata, GtfsFeedContainer feedContainer, GtfsFeedLoader loader) { final long endNanos = System.nanoTime(); List> multiFileValidatorsWithParsingErrors = loader.getMultiFileValidatorsWithParsingErrors(); List> singleFileValidatorsWithParsingErrors = - anyTableLoader.getValidatorsWithParsingErrors(); + TableLoader.getValidatorsWithParsingErrors(); // In theory single entity validators do not depend on files so there should not be any of these // with parsing errors List> singleEntityValidatorsWithParsingErrors = - anyTableLoader.getSingleEntityValidatorsWithParsingErrors(); + TableLoader.getSingleEntityValidatorsWithParsingErrors(); if (!singleFileValidatorsWithParsingErrors.isEmpty() || !singleEntityValidatorsWithParsingErrors.isEmpty() || !multiFileValidatorsWithParsingErrors.isEmpty()) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java similarity index 80% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java index 187da30894..dd77100dbb 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java @@ -3,6 +3,9 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * This class contains the information from one feature in the geojson file. + */ public final class GtfsGeojsonFeature implements GtfsEntity { public static final String FILENAME = "locations.geojson"; @@ -12,6 +15,7 @@ public final class GtfsGeojsonFeature implements GtfsEntity { public GtfsGeojsonFeature() {} + // TODO: Change the interface hierarchy so we dont need this. It's not relevant for geojson @Override public int csvRowNumber() { return 0; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java similarity index 100% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java similarity index 91% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index b8ca7005b6..ef930d021e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -29,4 +29,8 @@ public Class getEntityClass() { public String gtfsFilename() { return "locations.geojson"; } + + public TableLoader getTableLoader() { + return new JsonFileLoader(); + } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java similarity index 90% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java index c39816b0b1..bfa18cbcf9 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -13,16 +13,18 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; -public class JsonFileLoader { +public class JsonFileLoader extends TableLoader { - public static GtfsGeojsonFeaturesContainer load( - GtfsGeojsonFileDescriptor fileDescriptor, + @Override + public GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, ValidatorProvider validatorProvider, InputStream inputStream, NoticeContainer noticeContainer) { + GtfsGeojsonFileDescriptor geojsonFileDescriptor = (GtfsGeojsonFileDescriptor) fileDescriptor; try { List entities = extractFeaturesFromStream(inputStream, noticeContainer); - return fileDescriptor.createContainerForEntities(entities, noticeContainer); + return geojsonFileDescriptor.createContainerForEntities(entities, noticeContainer); } catch (IOException ioex) { noticeContainer.addSystemError(new IOError(ioex)); return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); @@ -81,4 +83,5 @@ public static GtfsGeojsonFeature extractFeature( } return gtfsGeojsonFeature; } + } From e3dbb2be21ff383111efda9db91b7409d5d01369 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 20 Sep 2024 14:41:45 -0400 Subject: [PATCH 18/28] Added a validator for the duplicate feature id in locations.. --- .../notice/GeojsonDuplicateKeyNotice.java | 42 +++++++++++++++++++ .../gtfsvalidator/table/AnyTableLoader.java | 1 - .../gtfsvalidator/table/TableLoader.java | 1 - .../table/GtfsGeojsonFeature.java | 4 +- .../table/GtfsGeojsonFeaturesContainer.java | 24 +++++++---- .../gtfsvalidator/table/JsonFileLoader.java | 1 - ...ojsonFeatureUniqueLocationIdValidator.java | 2 +- 7 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java new file mode 100644 index 0000000000..cfa6bc141c --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mobilitydata.gtfsvalidator.notice; + +import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS; +import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; + +/** + * Duplicated feature id. + * + *

One of the feature id of the locations.geojson file appears twice or more in the file. + */ +@GtfsValidationNotice(severity = ERROR, sections = @SectionRefs(FILE_REQUIREMENTS)) +public class GeojsonDuplicateKeyNotice extends ValidationNotice { + + /** The name of the faulty file. */ + private final String filename; + + /** The feature that is duplicated. */ + private final String featureId; + + public GeojsonDuplicateKeyNotice(String filename, String featureId) { + this.filename = filename; + this.featureId = featureId; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java index daae71d285..2d4e026b71 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java @@ -184,5 +184,4 @@ private void logFieldCacheStats( } } } - } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java index 7eb9aecfbf..6ba19c3666 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; - import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice; import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java index dd77100dbb..874f39e847 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java @@ -3,9 +3,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -/** - * This class contains the information from one feature in the geojson file. - */ +/** This class contains the information from one feature in the geojson file. */ public final class GtfsGeojsonFeature implements GtfsEntity { public static final String FILENAME = "locations.geojson"; diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index a87e9802c6..b71867d39f 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -1,3 +1,18 @@ +/* + * Copyright 2024 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.mobilitydata.gtfsvalidator.table; import java.util.ArrayList; @@ -5,7 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.mobilitydata.gtfsvalidator.notice.DuplicateKeyNotice; +import org.mobilitydata.gtfsvalidator.notice.GeojsonDuplicateKeyNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; public class GtfsGeojsonFeaturesContainer @@ -58,12 +73,7 @@ private void setupIndices(NoticeContainer noticeContainer) { GtfsGeojsonFeature oldEntity = byLocationIdMap.getOrDefault(newEntity.locationId(), null); if (oldEntity != null) { noticeContainer.addValidationNotice( - new DuplicateKeyNotice( - gtfsFilename(), - newEntity.csvRowNumber(), - oldEntity.csvRowNumber(), - GtfsGeojsonFeature.LOCATION_ID_FIELD_NAME, - newEntity.locationId())); + new GeojsonDuplicateKeyNotice(gtfsFilename(), newEntity.locationId())); } else { byLocationIdMap.put(newEntity.locationId(), newEntity); } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java index bfa18cbcf9..f754411ed7 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java @@ -83,5 +83,4 @@ public static GtfsGeojsonFeature extractFeature( } return gtfsGeojsonFeature; } - } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java index 5bbacdfe6d..fead6244ae 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2024 MobilityData * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From bffef74b47d828fc46c7f9148f55598f0324c9a3 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:30:54 -0400 Subject: [PATCH 19/28] add GtfsLocationsSchema and related classes --- .../table/GtfsLocationsGeoJsonTypeEnum.java | 22 ++++ .../table/GtfsLocationsSchema.java | 107 ++++++++++++++++++ .../table/locations/GtfsLocations.java | 10 ++ .../locations/GtfsLocationsDescriptor.java | 47 ++++++++ .../locations/GtfsLocationsJsonContainer.java | 44 +++++++ .../gtfsvalidator/annotation/GtfsJson.java | 39 +++++++ .../annotation/GtfsJsonEntity.java | 15 +++ .../processor/GtfsAnnotationProcessor.java | 5 +- 8 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java create mode 100644 model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java create mode 100644 model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java new file mode 100644 index 0000000000..5d3074c159 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.table; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsEnumValue; + +@GtfsEnumValue(name = "FeatureCollection", value = 0) +public interface GtfsLocationsGeoJsonTypeEnum {} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java new file mode 100644 index 0000000000..6e2fb9f243 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java @@ -0,0 +1,107 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.util.List; +import org.mobilitydata.gtfsvalidator.annotation.GtfsJson; +import org.mobilitydata.gtfsvalidator.annotation.GtfsJsonEntity; + +enum GeometryType { + POLYGON, + MULTIPOLYGON +} + +abstract class BaseGeometry { + protected GeometryType type; + + public BaseGeometry(GeometryType type) { + this.type = type; + } + + public GeometryType getType() { + return type; + } + + public void setType(GeometryType type) { + this.type = type; + } +} + +class Polygon extends BaseGeometry { + private List>> coordinates; // A list of rings, each a list of coordinate pairs + + public Polygon(List>> coordinates) { + super(GeometryType.POLYGON); + this.coordinates = coordinates; + } + + public List>> getCoordinates() { + return coordinates; + } + + public void setCoordinates(List>> coordinates) { + this.coordinates = coordinates; + } +} + +class MultiPolygon extends BaseGeometry { + private List>>> coordinates; // A list of polygons, each a list of rings + + public MultiPolygon(List>>> coordinates) { + super(GeometryType.MULTIPOLYGON); + this.coordinates = coordinates; + } + + public List>>> getCoordinates() { + return coordinates; + } + + public void setCoordinates(List>>> coordinates) { + this.coordinates = coordinates; + } +} + +class Feature { + private String id; + private BaseGeometry geometry; + private Properties properties; + + public Feature(String id, BaseGeometry geometry, Properties properties) { + this.id = id; + this.geometry = geometry; + this.properties = properties; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public BaseGeometry getGeometry() { + return geometry; + } + + public void setGeometry(BaseGeometry geometry) { + this.geometry = geometry; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } +} + +class Properties {} + +@GtfsJson("locations.geojson") +public interface GtfsLocationsSchema extends GtfsEntity { + // @FieldType(GtfsLocationsGeoJsonTypeEnum.ID) + String type(); + + @GtfsJsonEntity(value = "features", clazz = Feature.class) + List features(); +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java new file mode 100644 index 0000000000..6c06e544e6 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java @@ -0,0 +1,10 @@ +package org.mobilitydata.gtfsvalidator.table.locations; + +import org.mobilitydata.gtfsvalidator.table.GtfsEntity; + +public class GtfsLocations implements GtfsEntity { + @Override + public int csvRowNumber() { + return 0; + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java new file mode 100644 index 0000000000..17e2f57cca --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java @@ -0,0 +1,47 @@ +package org.mobilitydata.gtfsvalidator.table.locations; + +import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.table.*; + +public class GtfsLocationsDescriptor extends GtfsJsonDescriptor { + + @Override + public GtfsJsonContainer createContainerForEntities( + List entities, NoticeContainer noticeContainer) { + return GtfsLocationsDescriptor.forEntities(this, entities, noticeContainer); + } + + /** Creates a table with entities */ + public static GtfsLocationsJsonContainer forEntities( + GtfsLocationsDescriptor descriptor, + List entities, + NoticeContainer noticeContainer) { + // TODO review indices with the notice container + return new GtfsLocationsJsonContainer(descriptor, entities); + } + + public GtfsLocationsDescriptor() { + super(); + } + + @Override + public GtfsJsonContainer createContainerForInvalidStatus(TableStatus tableStatus) { + return new GtfsLocationsJsonContainer(this, tableStatus); + } + + @Override + public boolean isRecommended() { + return false; + } + + @Override + public Class getEntityClass() { + return null; + } + + @Override + public String gtfsFilename() { + return "locations.geojson"; + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java new file mode 100644 index 0000000000..04255eeb48 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java @@ -0,0 +1,44 @@ +package org.mobilitydata.gtfsvalidator.table.locations; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.mobilitydata.gtfsvalidator.table.GtfsJsonContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; + +public class GtfsLocationsJsonContainer + extends GtfsJsonContainer { + + private final List entities; + + public GtfsLocationsJsonContainer( + GtfsLocationsDescriptor descriptor, List entities) { + super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS); + this.entities = entities; + } + + public GtfsLocationsJsonContainer(GtfsLocationsDescriptor descriptor, TableStatus tableStatus) { + super(descriptor, tableStatus); + this.entities = new ArrayList<>(); + } + + @Override + public Class getEntityClass() { + return null; + } + + @Override + public List getEntities() { + return List.of(); + } + + @Override + public String gtfsFilename() { + return ""; + } + + @Override + public Optional byTranslationKey(String recordId, String recordSubId) { + return Optional.empty(); + } +} diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java new file mode 100644 index 0000000000..dce0723493 --- /dev/null +++ b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotates an interface that defines schema for a single GTFS JSON, such as "locations.geojson". + * + *

Example. + * + * 

+ *   {@literal @}GtfsJson("locations.geojson")
+ *   public interface GtfsLocationsSchema extends GtfsEntity {
+ *   }
+ *
+ */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface GtfsJson { + String value(); +} diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java new file mode 100644 index 0000000000..ff15fcc7f9 --- /dev/null +++ b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java @@ -0,0 +1,15 @@ +package org.mobilitydata.gtfsvalidator.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Annotates a field that defines entities for a GTFS JSON schema. */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface GtfsJsonEntity { + String value(); + + Class clazz(); +} diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java index ce1c4504cc..210f5178ca 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java @@ -37,10 +37,7 @@ import javax.lang.model.element.Element; import javax.lang.model.element.TypeElement; import javax.tools.Diagnostic; -import org.mobilitydata.gtfsvalidator.annotation.GtfsEnumValue; -import org.mobilitydata.gtfsvalidator.annotation.GtfsEnumValues; -import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; +import org.mobilitydata.gtfsvalidator.annotation.*; /** * Processor that generates data classes, loaders and validators based on annotations on GTFS schema From c90bd7e22768888a5c1092c0b2711c17293a6e0c Mon Sep 17 00:00:00 2001 From: jcpitre Date: Mon, 23 Sep 2024 22:38:10 -0400 Subject: [PATCH 20/28] Started using the Schema so the rules.hyml will be generated properly. --- .../UniqueLocationIdViolationNotice.java | 53 ------------------- .../notice/schema/NoticeSchemaGenerator.java | 12 ++++- .../notice/GeojsonDuplicateKeyNotice.java | 6 ++- .../table/GtfsLocationsGeoJsonTypeEnum.java | 22 -------- .../table/locations/GtfsLocations.java | 10 ---- .../locations/GtfsLocationsDescriptor.java | 47 ---------------- .../locations/GtfsLocationsJsonContainer.java | 44 --------------- ...ojsonFeatureUniqueLocationIdValidator.java | 48 ++++++++++++++++- 8 files changed, 63 insertions(+), 179 deletions(-) delete mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java rename {core => main}/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java (86%) delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java deleted file mode 100644 index eab5c20a9d..0000000000 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/UniqueLocationIdViolationNotice.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.mobilitydata.gtfsvalidator.notice; - -import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS; -import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; - -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; - -/** - * Feature id from locations.geojson already used. - * - *

The id of one of the features of the locations.geojson file already exists in stops.txt or - * location_groups.txt - */ -@GtfsValidationNotice(severity = ERROR, sections = @SectionRefs(FILE_REQUIREMENTS)) -public class UniqueLocationIdViolationNotice extends ValidationNotice { - - /** The id that already exists. */ - private final String id; - - /** The name of the file that already has this id. */ - private final String fileWithIdAlreadyPresent; - - /** The name of the field that contains this id. */ - private final String fieldNameInFile; - - /** The row of the record in the file where the id is already present. */ - private final int csvRowNumber; - - public UniqueLocationIdViolationNotice( - String id, String fileWithIdAlreadyPresent, String fieldNameInFile, int csvRowNumber) { - - this.id = id; - this.fileWithIdAlreadyPresent = fileWithIdAlreadyPresent; - this.fieldNameInFile = fieldNameInFile; - this.csvRowNumber = csvRowNumber; - } -} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java index 82d637755d..920dae583f 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.TreeMap; import java.util.logging.Level; +import org.mobilitydata.gtfsvalidator.annotation.GtfsJson; import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef; @@ -125,7 +126,11 @@ public static NoticeDocComments loadComments(Class noticeClass) { private static ReferencesSchema generateReferences(GtfsValidationNotice noticeAnnotation) { ReferencesSchema schema = new ReferencesSchema(); Arrays.stream(noticeAnnotation.files().value()) - .map(NoticeSchemaGenerator::getFileIdForTableClass) + .map( + fileClass -> { + Optional fileId = getFileIdForTableClass(fileClass); + return fileId.or(() -> getFileIdForJsonClass(fileClass)); + }) .flatMap(Optional::stream) .forEach(schema::addFileReference); Arrays.stream(noticeAnnotation.bestPractices().value()) @@ -146,6 +151,11 @@ private static Optional getFileIdForTableClass(Class getFileIdForJsonClass(Class tableClass) { + GtfsJson table = tableClass.getAnnotation(GtfsJson.class); + return Optional.ofNullable(table).map(GtfsJson::value); + } + private static UrlReference convertUrlRef(UrlRef ref) { return new UrlReference(ref.label(), ref.url()); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java similarity index 86% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java index cfa6bc141c..a2f4de82de 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java @@ -20,13 +20,17 @@ import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; +import org.mobilitydata.gtfsvalidator.table.GtfsLocationsSchema; /** * Duplicated feature id. * *

One of the feature id of the locations.geojson file appears twice or more in the file. */ -@GtfsValidationNotice(severity = ERROR, sections = @SectionRefs(FILE_REQUIREMENTS)) +@GtfsValidationNotice( + severity = ERROR, + files = @GtfsValidationNotice.FileRefs(GtfsLocationsSchema.class), + sections = @SectionRefs(FILE_REQUIREMENTS)) public class GeojsonDuplicateKeyNotice extends ValidationNotice { /** The name of the faulty file. */ diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java deleted file mode 100644 index 5d3074c159..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsGeoJsonTypeEnum.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.mobilitydata.gtfsvalidator.table; - -import org.mobilitydata.gtfsvalidator.annotation.GtfsEnumValue; - -@GtfsEnumValue(name = "FeatureCollection", value = 0) -public interface GtfsLocationsGeoJsonTypeEnum {} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java deleted file mode 100644 index 6c06e544e6..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocations.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.mobilitydata.gtfsvalidator.table.locations; - -import org.mobilitydata.gtfsvalidator.table.GtfsEntity; - -public class GtfsLocations implements GtfsEntity { - @Override - public int csvRowNumber() { - return 0; - } -} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java deleted file mode 100644 index 17e2f57cca..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsDescriptor.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.mobilitydata.gtfsvalidator.table.locations; - -import java.util.List; -import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -import org.mobilitydata.gtfsvalidator.table.*; - -public class GtfsLocationsDescriptor extends GtfsJsonDescriptor { - - @Override - public GtfsJsonContainer createContainerForEntities( - List entities, NoticeContainer noticeContainer) { - return GtfsLocationsDescriptor.forEntities(this, entities, noticeContainer); - } - - /** Creates a table with entities */ - public static GtfsLocationsJsonContainer forEntities( - GtfsLocationsDescriptor descriptor, - List entities, - NoticeContainer noticeContainer) { - // TODO review indices with the notice container - return new GtfsLocationsJsonContainer(descriptor, entities); - } - - public GtfsLocationsDescriptor() { - super(); - } - - @Override - public GtfsJsonContainer createContainerForInvalidStatus(TableStatus tableStatus) { - return new GtfsLocationsJsonContainer(this, tableStatus); - } - - @Override - public boolean isRecommended() { - return false; - } - - @Override - public Class getEntityClass() { - return null; - } - - @Override - public String gtfsFilename() { - return "locations.geojson"; - } -} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java deleted file mode 100644 index 04255eeb48..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/locations/GtfsLocationsJsonContainer.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.mobilitydata.gtfsvalidator.table.locations; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import org.mobilitydata.gtfsvalidator.table.GtfsJsonContainer; -import org.mobilitydata.gtfsvalidator.table.TableStatus; - -public class GtfsLocationsJsonContainer - extends GtfsJsonContainer { - - private final List entities; - - public GtfsLocationsJsonContainer( - GtfsLocationsDescriptor descriptor, List entities) { - super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS); - this.entities = entities; - } - - public GtfsLocationsJsonContainer(GtfsLocationsDescriptor descriptor, TableStatus tableStatus) { - super(descriptor, tableStatus); - this.entities = new ArrayList<>(); - } - - @Override - public Class getEntityClass() { - return null; - } - - @Override - public List getEntities() { - return List.of(); - } - - @Override - public String gtfsFilename() { - return ""; - } - - @Override - public Optional byTranslationKey(String recordId, String recordSubId) { - return Optional.empty(); - } -} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java index fead6244ae..e051658454 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java @@ -16,15 +16,22 @@ package org.mobilitydata.gtfsvalidator.validator; +import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS; +import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; + import java.util.Optional; import javax.inject.Inject; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -import org.mobilitydata.gtfsvalidator.notice.UniqueLocationIdViolationNotice; +import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupsSchema; +import org.mobilitydata.gtfsvalidator.table.GtfsLocationsSchema; import org.mobilitydata.gtfsvalidator.table.GtfsStop; import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsStopTimeSchema; /** * Validates that the feature id from "locations.geojson" is not a duplicate of any stop_id from @@ -72,4 +79,43 @@ public void validate(NoticeContainer noticeContainer) { gtfsStop.csvRowNumber()))); } } + + /** + * Feature id from locations.geojson already used. + * + *

The id of one of the features of the locations.geojson file already exists in stops.txt or + * location_groups.txt + */ + @GtfsValidationNotice( + severity = ERROR, + files = + @GtfsValidationNotice.FileRefs({ + GtfsLocationsSchema.class, + GtfsStopTimeSchema.class, + GtfsLocationGroupsSchema.class + }), + sections = @GtfsValidationNotice.SectionRefs(FILE_REQUIREMENTS)) + public static class UniqueLocationIdViolationNotice extends ValidationNotice { + + /** The id that already exists. */ + private final String id; + + /** The name of the file that already has this id. */ + private final String fileWithIdAlreadyPresent; + + /** The name of the field that contains this id. */ + private final String fieldNameInFile; + + /** The row of the record in the file where the id is already present. */ + private final int csvRowNumber; + + public UniqueLocationIdViolationNotice( + String id, String fileWithIdAlreadyPresent, String fieldNameInFile, int csvRowNumber) { + + this.id = id; + this.fileWithIdAlreadyPresent = fileWithIdAlreadyPresent; + this.fieldNameInFile = fieldNameInFile; + this.csvRowNumber = csvRowNumber; + } + } } From b794c3a4903d409fa1a1cce7baf1c18819185a0f Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 24 Sep 2024 14:06:38 -0400 Subject: [PATCH 21/28] Added location_group_stops so the UniqueGeographyIdValidator is more complete. --- .../table/GtfsLocationGroupStopsSchema.java | 23 ++++++++++++ ...r.java => UniqueGeographyIdValidator.java} | 37 +++++++++++++------ 2 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java rename main/src/main/java/org/mobilitydata/gtfsvalidator/validator/{GtfsGeojsonFeatureUniqueLocationIdValidator.java => UniqueGeographyIdValidator.java} (72%) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java new file mode 100644 index 0000000000..837e6ef192 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java @@ -0,0 +1,23 @@ +package org.mobilitydata.gtfsvalidator.table; + +import org.mobilitydata.gtfsvalidator.annotation.FieldType; +import org.mobilitydata.gtfsvalidator.annotation.FieldTypeEnum; +import org.mobilitydata.gtfsvalidator.annotation.ForeignKey; +import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; +import org.mobilitydata.gtfsvalidator.annotation.Index; +import org.mobilitydata.gtfsvalidator.annotation.Required; + +@GtfsTable("location_group_stops.txt") +public interface GtfsLocationGroupStopsSchema extends GtfsEntity { + + @FieldType(FieldTypeEnum.ID) + @ForeignKey(table = "location_groups.txt", field = "location_group_id") + @Index + @Required + String locationGroupId(); + + @FieldType(FieldTypeEnum.ID) + @ForeignKey(table = "stops.txt", field = "stop_id") + @Required + String stopId(); +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java similarity index 72% rename from main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java index e051658454..4d86847077 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/GtfsGeojsonFeatureUniqueLocationIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java @@ -27,6 +27,8 @@ import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupStops; +import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupStopsTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupsSchema; import org.mobilitydata.gtfsvalidator.table.GtfsLocationsSchema; import org.mobilitydata.gtfsvalidator.table.GtfsStop; @@ -35,29 +37,28 @@ /** * Validates that the feature id from "locations.geojson" is not a duplicate of any stop_id from - * "stops.txt" or location_group_id from "location_groups.txt" + * "stops.txt" or location_group_id from "location_group_stops.txt" * - *

Generated notice: {@link UniqueLocationIdViolationNotice}. + *

Generated notice: {@link DuplicateGeographyIdNotice}. */ @GtfsValidator -public class GtfsGeojsonFeatureUniqueLocationIdValidator extends FileValidator { +public class UniqueGeographyIdValidator extends FileValidator { private final GtfsStopTableContainer stopTableContainer; // Remove this comment when the location_group_stops.txt file is added to the GTFS schema - // private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; + private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; private final GtfsGeojsonFeaturesContainer geojsonFeatureContainer; @Inject - GtfsGeojsonFeatureUniqueLocationIdValidator( + UniqueGeographyIdValidator( GtfsGeojsonFeaturesContainer geojsonFeatureContainer, - GtfsStopTableContainer stopTableContainer - // , GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer - ) { + GtfsStopTableContainer stopTableContainer, + GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer) { this.geojsonFeatureContainer = geojsonFeatureContainer; this.stopTableContainer = stopTableContainer; - // this.locationGroupStopsTableContainer = locationGroupStopsTableContainer; + this.locationGroupStopsTableContainer = locationGroupStopsTableContainer; } @Override @@ -72,12 +73,24 @@ public void validate(NoticeContainer noticeContainer) { stop.ifPresent( gtfsStop -> noticeContainer.addValidationNotice( - new UniqueLocationIdViolationNotice( + new DuplicateGeographyIdNotice( locationId, GtfsStop.FILENAME, GtfsStop.STOP_ID_FIELD_NAME, gtfsStop.csvRowNumber()))); + + // Result is a list since we have not specified unicity for that field (i.e. PrimaryKey) + var locationGroupStopList = locationGroupStopsTableContainer.byLocationGroupId(locationId); + if (!locationGroupStopList.isEmpty()) { + noticeContainer.addValidationNotice( + new DuplicateGeographyIdNotice( + locationId, + GtfsLocationGroupStops.FILENAME, + GtfsLocationGroupStops.LOCATION_GROUP_ID_FIELD_NAME, + locationGroupStopList.get(0).csvRowNumber())); + } } + // TODO: Add detection of duplicates between stops and location_group_stops } /** @@ -95,7 +108,7 @@ public void validate(NoticeContainer noticeContainer) { GtfsLocationGroupsSchema.class }), sections = @GtfsValidationNotice.SectionRefs(FILE_REQUIREMENTS)) - public static class UniqueLocationIdViolationNotice extends ValidationNotice { + public static class DuplicateGeographyIdNotice extends ValidationNotice { /** The id that already exists. */ private final String id; @@ -109,7 +122,7 @@ public static class UniqueLocationIdViolationNotice extends ValidationNotice { /** The row of the record in the file where the id is already present. */ private final int csvRowNumber; - public UniqueLocationIdViolationNotice( + public DuplicateGeographyIdNotice( String id, String fileWithIdAlreadyPresent, String fieldNameInFile, int csvRowNumber) { this.id = id; From 4badc74e48eabcfeb092f6ecf0e1d88343355ba1 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 26 Sep 2024 16:12:24 -0400 Subject: [PATCH 22/28] Modified according to PR comments. --- ...AnyTableLoader.java => CsvFileLoader.java} | 8 +- .../table/GtfsEntityContainer.java | 6 +- .../table/GtfsFeedContainer.java | 2 +- .../table/GtfsFileDescriptor.java | 2 +- .../gtfsvalidator/testing/LoadingHelper.java | 4 +- ...oaderTest.java => CsvTableLoaderTest.java} | 14 +-- ...otice.java => JsonDuplicateKeyNotice.java} | 14 ++- .../report/model/FeedMetadata.java | 2 +- ...FileLoader.java => GeoJsonFileLoader.java} | 4 +- .../table/GtfsGeojsonFeature.java | 16 +-- .../table/GtfsGeojsonFeaturesContainer.java | 11 +- .../table/GtfsGeojsonFileDescriptor.java | 2 +- .../validator/UniqueGeographyIdValidator.java | 12 +- .../validator/NoticeFieldsTest.java | 3 +- .../processor/notices/NoticesProcessor.java | 46 ++++---- .../tests/CurrencyAmountSchemaTest.java | 84 +++++++++++++ .../processor/tests/MixedCaseSchemaTest.java | 110 ++++++++++++++++++ 17 files changed, 267 insertions(+), 73 deletions(-) rename core/src/main/java/org/mobilitydata/gtfsvalidator/table/{AnyTableLoader.java => CsvFileLoader.java} (97%) rename core/src/test/java/org/mobilitydata/gtfsvalidator/table/{AnyTableLoaderTest.java => CsvTableLoaderTest.java} (96%) rename main/src/main/java/org/mobilitydata/gtfsvalidator/notice/{GeojsonDuplicateKeyNotice.java => JsonDuplicateKeyNotice.java} (79%) rename main/src/main/java/org/mobilitydata/gtfsvalidator/table/{JsonFileLoader.java => GeoJsonFileLoader.java} (95%) create mode 100644 processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java create mode 100644 processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java similarity index 97% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java index 2d4e026b71..d169da3cc1 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java @@ -24,13 +24,13 @@ import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; /** This class loads csv files specifically. */ -public final class AnyTableLoader extends TableLoader { +public final class CsvFileLoader extends TableLoader { - private AnyTableLoader() {} + private CsvFileLoader() {} // Create the singleton and add a method to obtain it - private static final AnyTableLoader INSTANCE = new AnyTableLoader(); + private static final CsvFileLoader INSTANCE = new CsvFileLoader(); - public static AnyTableLoader getInstance() { + public static CsvFileLoader getInstance() { return INSTANCE; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java index 1e7bf96d32..9c721a1d14 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java @@ -5,11 +5,11 @@ /** * This class is the parent of the containers holding table (csv) entities and containers holding - * geojson entities + * JSON entities * - * @param The entity for this container (e.g. GtfsCalendarDate or {@link GtfsGeojsonFeature} ) + * @param The entity for this container (e.g. GtfsCalendarDate or GtfsGeojsonFeature ) * @param The descriptor for the table for the container (e.g. GtfsCalendarDateTableDescriptor - * or {@link GtfsGeojsonFileDescriptor}) + * or GtfsGeojsonFileDescriptor) */ public abstract class GtfsEntityContainer { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java index 6731df5a49..967cfaedd8 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java @@ -22,7 +22,7 @@ /** * Container for a whole parsed GTFS feed with all its tables. * - *

The tables are kept as {@code GtfsContainer} instances. + *

The tables are kept as {@link GtfsEntityContainer} instances. */ public class GtfsFeedContainer { private final Map> tables = new HashMap<>(); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java index 8b06fa40ee..dae5f6b2c3 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java @@ -32,6 +32,6 @@ public void setRequired(boolean required) { } public TableLoader getTableLoader() { - return AnyTableLoader.getInstance(); + return CsvFileLoader.getInstance(); } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java index f89557fb9a..fbd5dfa80e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java @@ -26,7 +26,7 @@ import org.mobilitydata.gtfsvalidator.input.DateForValidation; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; -import org.mobilitydata.gtfsvalidator.table.AnyTableLoader; +import org.mobilitydata.gtfsvalidator.table.CsvFileLoader; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; @@ -70,6 +70,6 @@ public void setValidatorLoader(ValidatorLoader validatorLoader) { .setDateForValidation(new DateForValidation(dateForValidation)) .build(); ValidatorProvider provider = new DefaultValidatorProvider(context, validatorLoader); - return (Y) AnyTableLoader.getInstance().load(tableDescriptor, provider, in, noticeContainer); + return (Y) CsvFileLoader.getInstance().load(tableDescriptor, provider, in, noticeContainer); } } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/CsvTableLoaderTest.java similarity index 96% rename from core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java rename to core/src/test/java/org/mobilitydata/gtfsvalidator/table/CsvTableLoaderTest.java index 1c9d63a5d8..34cf2e1423 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/CsvTableLoaderTest.java @@ -30,7 +30,7 @@ import org.mockito.junit.MockitoRule; import org.mockito.quality.Strictness; -public class AnyTableLoaderTest { +public class CsvTableLoaderTest { @Rule public MockitoRule rule = MockitoJUnit.rule().strictness(Strictness.STRICT_STUBS); @Mock private GtfsTableContainer mockContainer; @@ -50,7 +50,7 @@ public void invalidInputStream() { .thenReturn(mockContainer); var loadedContainer = - AnyTableLoader.getInstance() + CsvFileLoader.getInstance() .load(testTableDescriptor, validatorProvider, null, loaderNotices); assertThat(validationNoticeTypes(loaderNotices)).containsExactly(CsvParsingFailedNotice.class); @@ -66,7 +66,7 @@ public void emptyInputStream() { InputStream csvInputStream = toInputStream(""); var loadedContainer = - AnyTableLoader.getInstance() + CsvFileLoader.getInstance() .load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) @@ -99,7 +99,7 @@ public void validate( when(validatorProvider.getTableHeaderValidator()).thenReturn(tableHeaderValidator); var loadedContainer = - AnyTableLoader.getInstance() + CsvFileLoader.getInstance() .load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()).containsExactly(headerValidationNotice); @@ -115,7 +115,7 @@ public void invalidRowLengthNotice() { InputStream inputStream = toInputStream("id,code\n" + "s1\n"); var loadedContainer = - AnyTableLoader.getInstance() + CsvFileLoader.getInstance() .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) @@ -136,7 +136,7 @@ public void parsableTableRows() { InputStream inputStream = toInputStream("id,stop_lat,_no_name_\n" + "s1, 23.00, no_value\n"); var loadedContainer = - AnyTableLoader.getInstance() + CsvFileLoader.getInstance() .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loadedContainer.getTableStatus()).isEqualTo(TableStatus.PARSABLE_HEADERS_AND_ROWS); @@ -172,7 +172,7 @@ public void missingRequiredField() { InputStream inputStream = toInputStream("id,code\n" + "s1,\n"); var loadedContainer = - AnyTableLoader.getInstance() + CsvFileLoader.getInstance() .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java similarity index 79% rename from main/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java index a2f4de82de..0047f4d33c 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/GeojsonDuplicateKeyNotice.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java @@ -31,16 +31,20 @@ severity = ERROR, files = @GtfsValidationNotice.FileRefs(GtfsLocationsSchema.class), sections = @SectionRefs(FILE_REQUIREMENTS)) -public class GeojsonDuplicateKeyNotice extends ValidationNotice { +public class JsonDuplicateKeyNotice extends ValidationNotice { /** The name of the faulty file. */ private final String filename; - /** The feature that is duplicated. */ - private final String featureId; + /** The field that is duplicated. */ + private final String fieldName; - public GeojsonDuplicateKeyNotice(String filename, String featureId) { + /** The duplicate value of the field. */ + private final String fieldValue; + + public JsonDuplicateKeyNotice(String filename, String fieldName, String fieldValue) { this.filename = filename; - this.featureId = featureId; + this.fieldName = fieldName; + this.fieldValue = fieldValue; } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index b38d5cd8df..250744e9d6 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -117,7 +117,7 @@ private void setCount( var table = feedContainer.getTableForFilename(fileName); this.counts.put( countName, - table.map(GtfsContainer -> loadUniqueCount(GtfsContainer, clazz, idExtractor)).orElse(0)); + table.map(gtfsContainer -> loadUniqueCount(gtfsContainer, clazz, idExtractor)).orElse(0)); } private int loadUniqueCount( diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java similarity index 95% rename from main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java rename to main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java index f754411ed7..146b0ca451 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/JsonFileLoader.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java @@ -13,7 +13,7 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; -public class JsonFileLoader extends TableLoader { +public class GeoJsonFileLoader extends TableLoader { @Override public GtfsEntityContainer load( @@ -60,7 +60,7 @@ public static GtfsGeojsonFeature extractFeature( } if (featureObject.has("id")) { gtfsGeojsonFeature = new GtfsGeojsonFeature(); - gtfsGeojsonFeature.setLocationId(featureObject.get("id").getAsString()); + gtfsGeojsonFeature.setFeatureId(featureObject.get("id").getAsString()); } else { // Add a notice because id is required } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java index 874f39e847..76c5a33bd5 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java @@ -7,9 +7,9 @@ public final class GtfsGeojsonFeature implements GtfsEntity { public static final String FILENAME = "locations.geojson"; - public static final String LOCATION_ID_FIELD_NAME = "location_id"; + public static final String FEATURE_ID_FIELD_NAME = "feature_id"; - private String locationId; // The id of a feature in the GeoJSON file. + private String featureId; // The id of a feature in the GeoJSON file. public GtfsGeojsonFeature() {} @@ -20,15 +20,15 @@ public int csvRowNumber() { } @Nonnull - public String locationId() { - return locationId; + public String featureId() { + return featureId; } - public boolean hasLocationId() { - return locationId != null; + public boolean hasFeatureId() { + return featureId != null; } - public void setLocationId(@Nullable String locationId) { - this.locationId = locationId; + public void setFeatureId(@Nullable String featureId) { + this.featureId = featureId; } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index b71867d39f..70cdb9726d 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.mobilitydata.gtfsvalidator.notice.GeojsonDuplicateKeyNotice; +import org.mobilitydata.gtfsvalidator.notice.JsonDuplicateKeyNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; public class GtfsGeojsonFeaturesContainer @@ -67,15 +67,16 @@ public Optional byTranslationKey(String recordId, String rec private void setupIndices(NoticeContainer noticeContainer) { for (GtfsGeojsonFeature newEntity : entities) { - if (!newEntity.hasLocationId()) { + if (!newEntity.hasFeatureId()) { continue; } - GtfsGeojsonFeature oldEntity = byLocationIdMap.getOrDefault(newEntity.locationId(), null); + GtfsGeojsonFeature oldEntity = byLocationIdMap.getOrDefault(newEntity.featureId(), null); if (oldEntity != null) { noticeContainer.addValidationNotice( - new GeojsonDuplicateKeyNotice(gtfsFilename(), newEntity.locationId())); + new JsonDuplicateKeyNotice( + gtfsFilename(), GtfsGeojsonFeature.FEATURE_ID_FIELD_NAME, newEntity.featureId())); } else { - byLocationIdMap.put(newEntity.locationId(), newEntity); + byLocationIdMap.put(newEntity.featureId(), newEntity); } } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index ef930d021e..37b757d7f9 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -31,6 +31,6 @@ public String gtfsFilename() { } public TableLoader getTableLoader() { - return new JsonFileLoader(); + return new GeoJsonFileLoader(); } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java index 4d86847077..f4079c66f6 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java @@ -64,27 +64,27 @@ public class UniqueGeographyIdValidator extends FileValidator { @Override public void validate(NoticeContainer noticeContainer) { for (GtfsGeojsonFeature json : geojsonFeatureContainer.getEntities()) { - String locationId = json.locationId(); - if (locationId.isEmpty()) { + String featureId = json.featureId(); + if (featureId.isEmpty()) { continue; } - Optional stop = stopTableContainer.byStopId(locationId); + Optional stop = stopTableContainer.byStopId(featureId); stop.ifPresent( gtfsStop -> noticeContainer.addValidationNotice( new DuplicateGeographyIdNotice( - locationId, + featureId, GtfsStop.FILENAME, GtfsStop.STOP_ID_FIELD_NAME, gtfsStop.csvRowNumber()))); // Result is a list since we have not specified unicity for that field (i.e. PrimaryKey) - var locationGroupStopList = locationGroupStopsTableContainer.byLocationGroupId(locationId); + var locationGroupStopList = locationGroupStopsTableContainer.byLocationGroupId(featureId); if (!locationGroupStopList.isEmpty()) { noticeContainer.addValidationNotice( new DuplicateGeographyIdNotice( - locationId, + featureId, GtfsLocationGroupStops.FILENAME, GtfsLocationGroupStops.LOCATION_GROUP_ID_FIELD_NAME, locationGroupStopList.get(0).csvRowNumber())); diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java index f66f38c08d..fe26cba7b4 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java @@ -49,7 +49,7 @@ public void testNoticeClassFieldNames() { "Is this test failing? That likely means you've added a new field name to a " + "`Notice` that hasn't been used before. See `NoticeFieldsTest` for instructions.") .that(discoverValidationNoticeFieldNames()) - // Keep the list of field names is in sorted order. + // Keep the list of field names in sorted order. .containsExactly( "actual", "actualDistanceBetweenShapePoints", @@ -88,7 +88,6 @@ public void testNoticeClassFieldNames() { "expectedRouteId", "fareMediaId1", "fareMediaId2", - "featureId", "feedEndDate", "feedLang", "fieldName", diff --git a/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java b/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java index 0211b88787..5b489a1822 100644 --- a/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java +++ b/processor/notices/src/main/java/org/mobilitydata/gtfsvalidator/processor/notices/NoticesProcessor.java @@ -52,35 +52,31 @@ public SourceVersion getSupportedSourceVersion() { @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { - try { - for (TypeElement element : - typesIn(roundEnv.getElementsAnnotatedWith(GtfsValidationNotice.class))) { - Optional comments = docCommentsFactory.create(element); - if (comments.isEmpty()) { - continue; - } + for (TypeElement element : + typesIn(roundEnv.getElementsAnnotatedWith(GtfsValidationNotice.class))) { + Optional comments = docCommentsFactory.create(element); + if (comments.isEmpty()) { + continue; + } - PackageElement packageElement = processingEnv.getElementUtils().getPackageOf(element); - String resourceName = NoticeDocComments.getResourceNameForTypeElement(element); + PackageElement packageElement = processingEnv.getElementUtils().getPackageOf(element); + String resourceName = NoticeDocComments.getResourceNameForTypeElement(element); - try { - FileObject resource = - processingEnv - .getFiler() - .createResource( - StandardLocation.CLASS_OUTPUT, - packageElement.getQualifiedName(), - resourceName, - element); - try (Writer writer = resource.openWriter()) { - GSON.toJson(comments.get(), writer); - } - } catch (IOException e) { - throw new RuntimeException(e); + try { + FileObject resource = + processingEnv + .getFiler() + .createResource( + StandardLocation.CLASS_OUTPUT, + packageElement.getQualifiedName(), + resourceName, + element); + try (Writer writer = resource.openWriter()) { + GSON.toJson(comments.get(), writer); } + } catch (IOException e) { + throw new RuntimeException(e); } - } catch (Exception e) { - System.out.println("Error: " + e); } return false; } diff --git a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java new file mode 100644 index 0000000000..a2eafc802f --- /dev/null +++ b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/CurrencyAmountSchemaTest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mobilitydata.gtfsvalidator.processor.tests; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import java.math.BigDecimal; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mobilitydata.gtfsvalidator.notice.InvalidCurrencyAmountNotice; +import org.mobilitydata.gtfsvalidator.table.CurrencyAmountTableDescriptor; +import org.mobilitydata.gtfsvalidator.testing.LoadingHelper; +import org.mobilitydata.gtfsvalidator.validator.CurrencyAmountCurrencyAmountValidator; +import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader; +import org.mobilitydata.gtfsvalidator.validator.ValidatorLoaderException; + +@RunWith(JUnit4.class) +public class CurrencyAmountSchemaTest { + + private CurrencyAmountTableDescriptor tableDescriptor; + private LoadingHelper helper; + + @Before + public void setup() throws ValidatorLoaderException { + tableDescriptor = new CurrencyAmountTableDescriptor(); + helper = new LoadingHelper(); + helper.setValidatorLoader( + ValidatorLoader.createForClasses( + ImmutableList.of(CurrencyAmountCurrencyAmountValidator.class))); + } + + @Test + public void testValidCurrencyUSD() throws ValidatorLoaderException { + + helper.load(tableDescriptor, "amount,currency", "1.50,USD"); + + assertThat(helper.getValidationNotices()).isEmpty(); + } + + @Test + public void testInvalidCurrencyUSD() throws ValidatorLoaderException { + helper.load(tableDescriptor, "amount,currency", "1.5,USD"); + + assertThat(helper.getValidationNotices()) + .containsExactly( + new InvalidCurrencyAmountNotice( + "currency_amount.txt", "amount", 2, new BigDecimal("1.5"))); + } + + @Test + public void testValidCurrencyISK() throws ValidatorLoaderException { + // Icelandic króna expects no digits after decimal separator. + helper.load(tableDescriptor, "amount,currency", "5,ISK"); + + assertThat(helper.getValidationNotices()).isEmpty(); + } + + @Test + public void testInvalidCurrencyISK() throws ValidatorLoaderException { + // Icelandic króna expects no digits after decimal separator. + helper.load(tableDescriptor, "amount,currency", "5.0,ISK"); + + assertThat(helper.getValidationNotices()) + .containsExactly( + new InvalidCurrencyAmountNotice( + "currency_amount.txt", "amount", 2, new BigDecimal("5.0"))); + } +} diff --git a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java new file mode 100644 index 0000000000..46a4b9a1ca --- /dev/null +++ b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/MixedCaseSchemaTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mobilitydata.gtfsvalidator.processor.tests; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.Collection; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mobilitydata.gtfsvalidator.notice.MixedCaseRecommendedFieldNotice; +import org.mobilitydata.gtfsvalidator.table.MixedCaseTest; +import org.mobilitydata.gtfsvalidator.table.MixedCaseTestTableDescriptor; +import org.mobilitydata.gtfsvalidator.testing.LoadingHelper; +import org.mobilitydata.gtfsvalidator.validator.MixedCaseTestMixedCaseValidator; +import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader; +import org.mobilitydata.gtfsvalidator.validator.ValidatorLoaderException; + +@RunWith(Parameterized.class) +public class MixedCaseSchemaTest { + private final String value; + private final boolean isValid; + + public MixedCaseSchemaTest(String value, boolean isValid) { + this.value = value; + this.isValid = isValid; + } + + private MixedCaseTestTableDescriptor tableDescriptor; + private LoadingHelper helper; + + @Before + public void setup() throws ValidatorLoaderException { + tableDescriptor = new MixedCaseTestTableDescriptor(); + helper = new LoadingHelper(); + helper.setValidatorLoader( + ValidatorLoader.createForClasses(ImmutableList.of(MixedCaseTestMixedCaseValidator.class))); + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList( + new Object[][] { + // valid values + {"Mixed-Case", true}, + {"Mixed_Case", true}, + {"Mixed Case", true}, + {"22222", true}, + {"A1", true}, + {"ZA112", true}, + {"301", true}, + {"RTE 30", true}, + {"급 행 12", true}, + {"급행12", true}, + {"東西線", true}, + {"101B", true}, + {"A14C", true}, + {"A14c", true}, + {"A14-C", true}, + {"A14_C", true}, + {"A14 C", true}, + {"Route 1", true}, + {"Route 1 Boulevard", true}, + {"ZA12", true}, + {"Avenue des Champs-Élysées", true}, + {"UPPERCASE", true}, + {"ROUTE 22", true}, + {"ROUTE 1", true}, + {"route 1 Boulevard", true}, + {"Sentences are ok with one mixed case word", true}, + {"MixedCaseButSingleWord", true}, + // invalid values + {"lowercase", false}, + {"snake_case", false}, + {"kebab-case", false}, + {"UPPER-CASE", false}, + {"lower case space", false}, + {"34broadst", false}, + }); + } + + @Test + public void testMixedCase() throws ValidatorLoaderException { + helper.load(tableDescriptor, MixedCaseTest.SOME_FIELD_FIELD_NAME, value); + if (isValid) { + assertThat(helper.getValidationNotices()).isEmpty(); + } else { + assertThat(helper.getValidationNotices()) + .containsExactly( + new MixedCaseRecommendedFieldNotice( + MixedCaseTest.FILENAME, MixedCaseTest.SOME_FIELD_FIELD_NAME, value, 2)); + } + } +} From 67504ab6b82dd8e69b16ce4d954db5b771b51539 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 26 Sep 2024 23:56:27 -0400 Subject: [PATCH 23/28] Removed validators and notices not slated for 6.0 --- .../notice/JsonDuplicateKeyNotice.java | 50 ------- .../table/GtfsGeojsonFeaturesContainer.java | 14 +- ...StopTimesGeographyIdPresenceValidator.java | 18 +-- .../validator/UniqueGeographyIdValidator.java | 134 ------------------ .../validator/NoticeFieldsTest.java | 3 - ...TimesGeographyIdPresenceValidatorTest.java | 7 + 6 files changed, 25 insertions(+), 201 deletions(-) delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java deleted file mode 100644 index 0047f4d33c..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/notice/JsonDuplicateKeyNotice.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2024 MobilityData - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.mobilitydata.gtfsvalidator.notice; - -import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS; -import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; - -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; -import org.mobilitydata.gtfsvalidator.table.GtfsLocationsSchema; - -/** - * Duplicated feature id. - * - *

One of the feature id of the locations.geojson file appears twice or more in the file. - */ -@GtfsValidationNotice( - severity = ERROR, - files = @GtfsValidationNotice.FileRefs(GtfsLocationsSchema.class), - sections = @SectionRefs(FILE_REQUIREMENTS)) -public class JsonDuplicateKeyNotice extends ValidationNotice { - - /** The name of the faulty file. */ - private final String filename; - - /** The field that is duplicated. */ - private final String fieldName; - - /** The duplicate value of the field. */ - private final String fieldValue; - - public JsonDuplicateKeyNotice(String filename, String fieldName, String fieldValue) { - this.filename = filename; - this.fieldName = fieldName; - this.fieldValue = fieldValue; - } -} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index 70cdb9726d..d878a4facf 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.mobilitydata.gtfsvalidator.notice.JsonDuplicateKeyNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; public class GtfsGeojsonFeaturesContainer @@ -71,13 +70,16 @@ private void setupIndices(NoticeContainer noticeContainer) { continue; } GtfsGeojsonFeature oldEntity = byLocationIdMap.getOrDefault(newEntity.featureId(), null); - if (oldEntity != null) { - noticeContainer.addValidationNotice( - new JsonDuplicateKeyNotice( - gtfsFilename(), GtfsGeojsonFeature.FEATURE_ID_FIELD_NAME, newEntity.featureId())); - } else { + if (oldEntity == null) { byLocationIdMap.put(newEntity.featureId(), newEntity); } + // TODO: Removed that code until the notice is supported. + // else { + // noticeContainer.addValidationNotice( + // new JsonDuplicateKeyNotice( + // gtfsFilename(), GtfsGeojsonFeature.FEATURE_ID_FIELD_NAME, + // newEntity.featureId())); + // } } } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java index 72538e1473..75f80db03d 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java @@ -51,14 +51,16 @@ public void validate(GtfsStopTime stopTime, NoticeContainer noticeContainer) { noticeContainer.addValidationNotice( new MissingRequiredFieldNotice( GtfsStopTime.FILENAME, stopTime.csvRowNumber(), GtfsStopTime.STOP_ID_FIELD_NAME)); - } else if (presenceCount > 1) { - // More than one geography ID is present, but only one is allowed - noticeContainer.addValidationNotice( - new ForbiddenGeographyIdNotice( - stopTime.csvRowNumber(), - stopTime.stopId(), - stopTime.locationGroupId(), - stopTime.locationId())); } + // TODO: Put this back once we are ready to publish this notice. + // else if (presenceCount > 1) { + // // More than one geography ID is present, but only one is allowed + // noticeContainer.addValidationNotice( + // new ForbiddenGeographyIdNotice( + // stopTime.csvRowNumber(), + // stopTime.stopId(), + // stopTime.locationGroupId(), + // stopTime.locationId())); + // } } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java deleted file mode 100644 index f4079c66f6..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/UniqueGeographyIdValidator.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2024 MobilityData - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.mobilitydata.gtfsvalidator.validator; - -import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.FILE_REQUIREMENTS; -import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; - -import java.util.Optional; -import javax.inject.Inject; -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; -import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; -import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; -import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeature; -import org.mobilitydata.gtfsvalidator.table.GtfsGeojsonFeaturesContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupStops; -import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupStopsTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsLocationGroupsSchema; -import org.mobilitydata.gtfsvalidator.table.GtfsLocationsSchema; -import org.mobilitydata.gtfsvalidator.table.GtfsStop; -import org.mobilitydata.gtfsvalidator.table.GtfsStopTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsStopTimeSchema; - -/** - * Validates that the feature id from "locations.geojson" is not a duplicate of any stop_id from - * "stops.txt" or location_group_id from "location_group_stops.txt" - * - *

Generated notice: {@link DuplicateGeographyIdNotice}. - */ -@GtfsValidator -public class UniqueGeographyIdValidator extends FileValidator { - private final GtfsStopTableContainer stopTableContainer; - - // Remove this comment when the location_group_stops.txt file is added to the GTFS schema - private final GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer; - - private final GtfsGeojsonFeaturesContainer geojsonFeatureContainer; - - @Inject - UniqueGeographyIdValidator( - GtfsGeojsonFeaturesContainer geojsonFeatureContainer, - GtfsStopTableContainer stopTableContainer, - GtfsLocationGroupStopsTableContainer locationGroupStopsTableContainer) { - this.geojsonFeatureContainer = geojsonFeatureContainer; - - this.stopTableContainer = stopTableContainer; - this.locationGroupStopsTableContainer = locationGroupStopsTableContainer; - } - - @Override - public void validate(NoticeContainer noticeContainer) { - for (GtfsGeojsonFeature json : geojsonFeatureContainer.getEntities()) { - String featureId = json.featureId(); - if (featureId.isEmpty()) { - continue; - } - - Optional stop = stopTableContainer.byStopId(featureId); - stop.ifPresent( - gtfsStop -> - noticeContainer.addValidationNotice( - new DuplicateGeographyIdNotice( - featureId, - GtfsStop.FILENAME, - GtfsStop.STOP_ID_FIELD_NAME, - gtfsStop.csvRowNumber()))); - - // Result is a list since we have not specified unicity for that field (i.e. PrimaryKey) - var locationGroupStopList = locationGroupStopsTableContainer.byLocationGroupId(featureId); - if (!locationGroupStopList.isEmpty()) { - noticeContainer.addValidationNotice( - new DuplicateGeographyIdNotice( - featureId, - GtfsLocationGroupStops.FILENAME, - GtfsLocationGroupStops.LOCATION_GROUP_ID_FIELD_NAME, - locationGroupStopList.get(0).csvRowNumber())); - } - } - // TODO: Add detection of duplicates between stops and location_group_stops - } - - /** - * Feature id from locations.geojson already used. - * - *

The id of one of the features of the locations.geojson file already exists in stops.txt or - * location_groups.txt - */ - @GtfsValidationNotice( - severity = ERROR, - files = - @GtfsValidationNotice.FileRefs({ - GtfsLocationsSchema.class, - GtfsStopTimeSchema.class, - GtfsLocationGroupsSchema.class - }), - sections = @GtfsValidationNotice.SectionRefs(FILE_REQUIREMENTS)) - public static class DuplicateGeographyIdNotice extends ValidationNotice { - - /** The id that already exists. */ - private final String id; - - /** The name of the file that already has this id. */ - private final String fileWithIdAlreadyPresent; - - /** The name of the field that contains this id. */ - private final String fieldNameInFile; - - /** The row of the record in the file where the id is already present. */ - private final int csvRowNumber; - - public DuplicateGeographyIdNotice( - String id, String fileWithIdAlreadyPresent, String fieldNameInFile, int csvRowNumber) { - - this.id = id; - this.fileWithIdAlreadyPresent = fileWithIdAlreadyPresent; - this.fieldNameInFile = fieldNameInFile; - this.csvRowNumber = csvRowNumber; - } - } -} diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java index 259cbac18b..c23a2f216a 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java @@ -94,7 +94,6 @@ public void testNoticeClassFieldNames() { "fieldName", "fieldName1", "fieldName2", - "fieldNameInFile", "fieldNames", "fieldType", "fieldValue", @@ -102,14 +101,12 @@ public void testNoticeClassFieldNames() { "fieldValue2", "fileNameA", "fileNameB", - "fileWithIdAlreadyPresent", "filename", "firstIndex", "geoDistanceToShape", "hasEntrance", "hasExit", "headerCount", - "id", "index", "intersection", "isBidirectional", diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java index d03d2be952..54d4b21249 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java @@ -51,6 +51,13 @@ public void OneGeographyIdShouldGenerateNothing() { } @Test + public void NoGeographyIdShouldGenerateNotice() { + assertThat(validationNoticesFor(new GtfsStopTime.Builder().setCsvRowNumber(2).build())) + .containsExactly(new MissingRequiredFieldNotice("stop_times.txt", 2, "stop_id")); + } + + // TODO: Put back when this notice is ready to be published. + // @Test public void MultipleGeographyIdShouldGenerateNotice() { assertThat( validationNoticesFor( From 3e6845c630e84b7f8ca95d9adca796e582fe250d Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 27 Sep 2024 09:41:53 -0400 Subject: [PATCH 24/28] Removed foreign key validations on locations_groups_stops since it's not part 6.0 --- .../gtfsvalidator/table/GtfsLocationGroupStopsSchema.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java index 837e6ef192..b8e661a684 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java @@ -2,7 +2,6 @@ import org.mobilitydata.gtfsvalidator.annotation.FieldType; import org.mobilitydata.gtfsvalidator.annotation.FieldTypeEnum; -import org.mobilitydata.gtfsvalidator.annotation.ForeignKey; import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; import org.mobilitydata.gtfsvalidator.annotation.Index; import org.mobilitydata.gtfsvalidator.annotation.Required; @@ -11,13 +10,14 @@ public interface GtfsLocationGroupStopsSchema extends GtfsEntity { @FieldType(FieldTypeEnum.ID) - @ForeignKey(table = "location_groups.txt", field = "location_group_id") + // TODO: Put back the foreign key annotation when ready to publish the notice + // @ForeignKey(table = "location_groups.txt", field = "location_group_id") @Index @Required String locationGroupId(); @FieldType(FieldTypeEnum.ID) - @ForeignKey(table = "stops.txt", field = "stop_id") + // @ForeignKey(table = "stops.txt", field = "stop_id") @Required String stopId(); } From 2a423cc75b1a50076803d9e82a3791bc9236d499 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 27 Sep 2024 11:52:38 -0400 Subject: [PATCH 25/28] Clean-up, added comments, removed unused classes. --- .../notice/schema/NoticeSchemaGenerator.java | 6 +- .../gtfsvalidator/table/CsvFileLoader.java | 2 + .../table/GtfsEntityContainer.java | 8 +- .../gtfsvalidator/table/GtfsFeedLoader.java | 15 +-- .../table/GtfsFileDescriptor.java | 8 ++ .../table/GtfsTableContainer.java | 5 +- .../gtfsvalidator/table/TableStatus.java | 4 +- .../validator/DefaultValidatorProvider.java | 6 +- .../validator/ValidatorProvider.java | 5 +- .../table/GeoJsonFileLoader.java | 6 +- .../table/GtfsGeojsonFeatureSchema.java | 13 +++ .../table/GtfsGeojsonFeaturesContainer.java | 4 + .../table/GtfsGeojsonFileDescriptor.java | 10 ++ .../table/GtfsLocationsSchema.java | 107 ------------------ .../gtfsvalidator/annotation/GtfsJson.java | 2 +- .../annotation/GtfsJsonEntity.java | 15 --- .../processor/GtfsAnnotationProcessor.java | 8 +- .../processor/TableDescriptorGenerator.java | 7 +- 18 files changed, 75 insertions(+), 156 deletions(-) create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java delete mode 100644 model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java index 920dae583f..63392bacd2 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java @@ -151,9 +151,9 @@ private static Optional getFileIdForTableClass(Class getFileIdForJsonClass(Class tableClass) { - GtfsJson table = tableClass.getAnnotation(GtfsJson.class); - return Optional.ofNullable(table).map(GtfsJson::value); + private static Optional getFileIdForJsonClass(Class entityClass) { + GtfsJson annotation = entityClass.getAnnotation(GtfsJson.class); + return Optional.ofNullable(annotation).map(GtfsJson::value); } private static UrlReference convertUrlRef(UrlRef ref) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java index d169da3cc1..e00baebcc8 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.mobilitydata.gtfsvalidator.notice.CsvParsingFailedNotice; import org.mobilitydata.gtfsvalidator.notice.EmptyFileNotice; @@ -30,6 +31,7 @@ private CsvFileLoader() {} // Create the singleton and add a method to obtain it private static final CsvFileLoader INSTANCE = new CsvFileLoader(); + @Nonnull public static CsvFileLoader getInstance() { return INSTANCE; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java index 9c721a1d14..fcaef33a9e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java @@ -4,12 +4,12 @@ import java.util.Optional; /** - * This class is the parent of the containers holding table (csv) entities and containers holding - * JSON entities + * This class is the parent of containers holding table (csv) entities and containers holding JSON + * entities * * @param The entity for this container (e.g. GtfsCalendarDate or GtfsGeojsonFeature ) - * @param The descriptor for the table for the container (e.g. GtfsCalendarDateTableDescriptor - * or GtfsGeojsonFileDescriptor) + * @param The descriptor for the file for the container (e.g. GtfsCalendarDateTableDescriptor or + * GtfsGeojsonFileDescriptor) */ public abstract class GtfsEntityContainer { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 999f10af7c..21dc38e98b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -117,16 +117,9 @@ public GtfsFeedContainer loadAndValidate( TableLoader tableLoader = tableDescriptor.getTableLoader(); try (InputStream inputStream = gtfsInput.getFile(filename)) { try { - if (tableLoader != null) { - tableContainer = - tableLoader.load( - tableDescriptor, validatorProvider, inputStream, loaderNotices); - } else { - logger.atSevere().log( - "Runtime exception table descriptor not supported: %s", - tableDescriptor.getClass().getName()); - throw new RuntimeException("Table descriptor is not a supported type"); - } + tableContainer = + tableLoader.load( + tableDescriptor, validatorProvider, inputStream, loaderNotices); } catch (RuntimeException e) { // This handler should prevent ExecutionException for // this thread. We catch an exception here for storing @@ -135,7 +128,7 @@ public GtfsFeedContainer loadAndValidate( loaderNotices.addSystemError(new RuntimeExceptionInLoaderError(filename, e)); // Since the file was not loaded successfully, we treat // it as missing for continuing validation. - // tableLoader = tableDescriptor.getTableLoader(); + tableContainer = tableLoader.loadMissingFile( tableDescriptor, validatorProvider, loaderNotices); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java index dae5f6b2c3..3a9193e503 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java @@ -1,5 +1,7 @@ package org.mobilitydata.gtfsvalidator.table; +import javax.annotation.Nonnull; + /** * This class provides some info about the different files within a GTFS dataset. Its children * relate to either a csv table or a geojson file. @@ -31,6 +33,12 @@ public void setRequired(boolean required) { this.required = required; } + /** + * Get the looder for the file described by this file descriptor. + * + * @return the appropriate file loader. + */ + @Nonnull public TableLoader getTableLoader() { return CsvFileLoader.getInstance(); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java index 0ea8258f80..3723b2b08e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java @@ -23,13 +23,12 @@ import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; /** - * Container for {@code GtfsEntity} instances for the whole GTFS table, e.g., stops.txt. + * Container for {@code GtfsEntity} instances coming from a CSV file. e.g., stops.txt. * *

Its subclasses are generated by annotation processor based on GTFS schema annotations. - * Instances of the subclasses are created by subclasses of {@code GtfsTableLoader} which are also - * generated by the processor. * * @param subclass of {@code GtfsEntity} + * @param subclass of {@code GtfsTableDescriptor} */ public abstract class GtfsTableContainer extends GtfsEntityContainer { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java index 417d01e772..0d88aaaf44 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java @@ -1,8 +1,8 @@ package org.mobilitydata.gtfsvalidator.table; /** - * Status of loading this table. This is includes parsing of the CSV file and validation of the - * single file, but does not include any cross-file validations. + * Status of loading this table. This includes parsing of the CSV file and validation of the single + * file, but does not include any cross-file validations. */ public enum TableStatus { /** The file is completely empty, i.e. it has no rows and even no headers. */ diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java index 32f55e6583..c5a1578670 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java @@ -21,7 +21,11 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; -import org.mobilitydata.gtfsvalidator.table.*; +import org.mobilitydata.gtfsvalidator.table.GtfsEntity; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader.ValidatorWithDependencyStatus; /** Default implementation of {@link ValidatorProvider}. */ diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java index cab1124841..2a9d754eca 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java @@ -18,7 +18,10 @@ import java.util.List; import java.util.function.Consumer; -import org.mobilitydata.gtfsvalidator.table.*; +import org.mobilitydata.gtfsvalidator.table.GtfsEntity; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; /** * Provider of all kinds of validators for fields, entities and files. diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java index 146b0ca451..ef532e2007 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java @@ -13,6 +13,7 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; +/** This class knows how to load a geojson file. */ public class GeoJsonFileLoader extends TableLoader { @Override @@ -31,7 +32,7 @@ public GtfsEntityContainer load( } } - public static List extractFeaturesFromStream( + public List extractFeaturesFromStream( InputStream inputStream, NoticeContainer noticeContainer) throws IOException { List features = new ArrayList<>(); try (InputStreamReader reader = new InputStreamReader(inputStream)) { @@ -47,8 +48,7 @@ public static List extractFeaturesFromStream( return features; } - public static GtfsGeojsonFeature extractFeature( - JsonElement feature, NoticeContainer noticeContainer) { + public GtfsGeojsonFeature extractFeature(JsonElement feature, NoticeContainer noticeContainer) { GtfsGeojsonFeature gtfsGeojsonFeature = null; if (feature.isJsonObject()) { JsonObject featureObject = feature.getAsJsonObject(); diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java new file mode 100644 index 0000000000..0a644ae7d2 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java @@ -0,0 +1,13 @@ +package org.mobilitydata.gtfsvalidator.table; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsJson; + +/** + * This class contains the information from one feature in the geojson file. Note that currently no + * class is autogenerated from this schema, contrarily to csv based entities. + */ +@GtfsJson("locations.geojson") +public interface GtfsGeojsonFeatureSchema extends GtfsEntity { + + String featureId(); +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java index d878a4facf..04c48d4909 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -22,6 +22,10 @@ import java.util.Optional; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +/** + * Container for geojson features. Contrarily to the csv containers, this class is not auto + * generated since we have only one such class. + */ public class GtfsGeojsonFeaturesContainer extends GtfsEntityContainer { diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index 37b757d7f9..aca4f3121c 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -1,10 +1,19 @@ package org.mobilitydata.gtfsvalidator.table; import java.util.List; +import javax.annotation.Nonnull; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +/** + * File descriptor for geojson file. Contrarily to the csv file descriptor, this class is not auto + * generated since we have only one such class. + */ public class GtfsGeojsonFileDescriptor extends GtfsFileDescriptor { + public GtfsGeojsonFileDescriptor() { + setRequired(false); + } + public GtfsGeojsonFeaturesContainer createContainerForEntities( List entities, NoticeContainer noticeContainer) { return new GtfsGeojsonFeaturesContainer(this, entities, noticeContainer); @@ -30,6 +39,7 @@ public String gtfsFilename() { return "locations.geojson"; } + @Nonnull public TableLoader getTableLoader() { return new GeoJsonFileLoader(); } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java deleted file mode 100644 index 6e2fb9f243..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationsSchema.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.mobilitydata.gtfsvalidator.table; - -import java.util.List; -import org.mobilitydata.gtfsvalidator.annotation.GtfsJson; -import org.mobilitydata.gtfsvalidator.annotation.GtfsJsonEntity; - -enum GeometryType { - POLYGON, - MULTIPOLYGON -} - -abstract class BaseGeometry { - protected GeometryType type; - - public BaseGeometry(GeometryType type) { - this.type = type; - } - - public GeometryType getType() { - return type; - } - - public void setType(GeometryType type) { - this.type = type; - } -} - -class Polygon extends BaseGeometry { - private List>> coordinates; // A list of rings, each a list of coordinate pairs - - public Polygon(List>> coordinates) { - super(GeometryType.POLYGON); - this.coordinates = coordinates; - } - - public List>> getCoordinates() { - return coordinates; - } - - public void setCoordinates(List>> coordinates) { - this.coordinates = coordinates; - } -} - -class MultiPolygon extends BaseGeometry { - private List>>> coordinates; // A list of polygons, each a list of rings - - public MultiPolygon(List>>> coordinates) { - super(GeometryType.MULTIPOLYGON); - this.coordinates = coordinates; - } - - public List>>> getCoordinates() { - return coordinates; - } - - public void setCoordinates(List>>> coordinates) { - this.coordinates = coordinates; - } -} - -class Feature { - private String id; - private BaseGeometry geometry; - private Properties properties; - - public Feature(String id, BaseGeometry geometry, Properties properties) { - this.id = id; - this.geometry = geometry; - this.properties = properties; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public BaseGeometry getGeometry() { - return geometry; - } - - public void setGeometry(BaseGeometry geometry) { - this.geometry = geometry; - } - - public Properties getProperties() { - return properties; - } - - public void setProperties(Properties properties) { - this.properties = properties; - } -} - -class Properties {} - -@GtfsJson("locations.geojson") -public interface GtfsLocationsSchema extends GtfsEntity { - // @FieldType(GtfsLocationsGeoJsonTypeEnum.ID) - String type(); - - @GtfsJsonEntity(value = "features", clazz = Feature.class) - List features(); -} diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java index dce0723493..d9ea7a108a 100644 --- a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java +++ b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2024 MobilityData * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java deleted file mode 100644 index ff15fcc7f9..0000000000 --- a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJsonEntity.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.mobilitydata.gtfsvalidator.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** Annotates a field that defines entities for a GTFS JSON schema. */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface GtfsJsonEntity { - String value(); - - Class clazz(); -} diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java index 210f5178ca..afa907421a 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsAnnotationProcessor.java @@ -37,7 +37,10 @@ import javax.lang.model.element.Element; import javax.lang.model.element.TypeElement; import javax.tools.Diagnostic; -import org.mobilitydata.gtfsvalidator.annotation.*; +import org.mobilitydata.gtfsvalidator.annotation.GtfsEnumValue; +import org.mobilitydata.gtfsvalidator.annotation.GtfsEnumValues; +import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; /** * Processor that generates data classes, loaders and validators based on annotations on GTFS schema @@ -48,9 +51,6 @@ public class GtfsAnnotationProcessor extends AbstractProcessor { private final Analyser analyser = new Analyser(); - public GtfsAnnotationProcessor() { - super(); - } /** * Sanitizes the result of {@link RoundEnvironment#getElementsAnnotatedWith}, which otherwise can * contain elements annotated with annotations of ERROR type. diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java index 187f7fbc71..62b01ac216 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java @@ -42,7 +42,12 @@ import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.parsing.FieldCache; import org.mobilitydata.gtfsvalidator.parsing.RowParser; -import org.mobilitydata.gtfsvalidator.table.*; +import org.mobilitydata.gtfsvalidator.table.GtfsColumnDescriptor; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityBuilder; +import org.mobilitydata.gtfsvalidator.table.GtfsFieldLoader; +import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.TableStatus; /** * Generates code for a GtfsTableDescriptor subclass for a specific GTFS table. From 63b7d621933e045003a15a7ff982011ecd901e6c Mon Sep 17 00:00:00 2001 From: jcpitre Date: Mon, 30 Sep 2024 16:59:01 -0400 Subject: [PATCH 26/28] Added some tests --- .../table/GeojsonFileLoader.java | 96 +++++++++++++++ .../table/GtfsGeojsonFileDescriptor.java | 2 +- .../table/GeojsonFileLoaderTest.java | 115 ++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java create mode 100644 main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java new file mode 100644 index 0000000000..472e4e64c5 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java @@ -0,0 +1,96 @@ +package org.mobilitydata.gtfsvalidator.table; + +import com.google.common.flogger.FluentLogger; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.IOError; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; + +/** This class knows how to load a geojson file. */ +public class GeojsonFileLoader extends TableLoader { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Override + public GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, + ValidatorProvider validatorProvider, + InputStream inputStream, + NoticeContainer noticeContainer) { + GtfsGeojsonFileDescriptor geojsonFileDescriptor = (GtfsGeojsonFileDescriptor) fileDescriptor; + try { + List entities = extractFeaturesFromStream(inputStream, noticeContainer); + return geojsonFileDescriptor.createContainerForEntities(entities, noticeContainer); + } catch (JsonParseException jpex) { + // TODO: Add a notice for malformed locations.geojson + logger.atSevere().withCause(jpex).log("Malformed JSON in locations.geojson"); + return geojsonFileDescriptor.createContainerForEntities(new ArrayList<>(), noticeContainer); + } catch (IOException ioex) { + noticeContainer.addSystemError(new IOError(ioex)); + return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); + } catch (Exception ex) { + logger.atSevere().withCause(ex).log("Error while loading locations.geojson"); + return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); + } + } + + public List extractFeaturesFromStream( + InputStream inputStream, NoticeContainer noticeContainer) throws IOException { + List features = new ArrayList<>(); + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); + JsonArray featuresArray = jsonObject.getAsJsonArray("features"); + for (JsonElement feature : featuresArray) { + GtfsGeojsonFeature gtfsGeojsonFeature = extractFeature(feature, noticeContainer); + if (gtfsGeojsonFeature != null) { + features.add(gtfsGeojsonFeature); + } + } + } + return features; + } + + public GtfsGeojsonFeature extractFeature(JsonElement feature, NoticeContainer noticeContainer) { + GtfsGeojsonFeature gtfsGeojsonFeature = null; + if (feature.isJsonObject()) { + JsonObject featureObject = feature.getAsJsonObject(); + if (featureObject.has("properties")) { + JsonObject properties = featureObject.getAsJsonObject("properties"); + // Add stop_name and stop_desc + } else { + // Add a notice because properties is required + } + if (featureObject.has("id")) { + gtfsGeojsonFeature = new GtfsGeojsonFeature(); + gtfsGeojsonFeature.setFeatureId(featureObject.get("id").getAsString()); + } else { + // Add a notice because id is required + } + + if (featureObject.has("geometry")) { + JsonObject geometry = featureObject.getAsJsonObject("geometry"); + if (geometry.has("type")) { + String type = geometry.get("type").getAsString(); + if (type.equals("Polygon")) { + // Extract the polygon + } else if (type.equals("Multipolygon")) { + // extract the multipolygon + } + } else { + // Add a notice because type is required + } + } else { + // Add a notice because geometry is required + } + } + return gtfsGeojsonFeature; + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java index aca4f3121c..ce96f087ca 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -41,6 +41,6 @@ public String gtfsFilename() { @Nonnull public TableLoader getTableLoader() { - return new GeoJsonFileLoader(); + return new GeojsonFileLoader(); } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java new file mode 100644 index 0000000000..17d81eb2e9 --- /dev/null +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2024 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.table; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; + +/** Runs GeojsonFileLoader on test json data. */ +@RunWith(JUnit4.class) +public class GeojsonFileLoaderTest { + + static String validGeojsonData; + + @BeforeClass + public static void setUpBeforeClass() { + // To make the json text clearer, use single quotes and replace them by double quotes before + // using + validGeojsonData = + String.join( + "\n", + "{", + " 'type': 'FeatureCollection',", + " 'features': [", + " {", + " 'id': 'id1',", + " 'type': 'Feature',", + " 'geometry': {", + " 'type': 'Point',", + " 'coordinates': [", + " [102.0, 0.0],", + " [103.0, 1.0],", + " [104.0, 0.0],", + " [105.0, 1.0]", + " ]", + " },", + " 'properties': {}", + " },", + " {", + " 'type': 'Feature',", + " 'id': 'id2',", + " 'geometry': {", + " 'type': 'Polygon',", + " 'coordinates': [", + " [", + " [100.0, 0.0],", + " [101.0, 0.0],", + " [101.0, 1.0],", + " [100.0, 1.0],", + " [100.0, 0.0]", + " ]", + " ]", + " },", + " 'properties': {}", + " }", + " ]", + "}"); + + validGeojsonData = validGeojsonData.replace("'", "\""); + } + + @Test + public void testGtfsGeojsonFileLoader() /*throws ValidatorLoaderException*/ { + + var container = createLoader(validGeojsonData); + var geojsonContainer = (GtfsGeojsonFeaturesContainer) container; + assertNotNull(container); + assertEquals( + "Test geojson file is not parsable", + container.getTableStatus(), + TableStatus.PARSABLE_HEADERS_AND_ROWS); + assertEquals(2, container.entityCount()); + assertEquals("id1", geojsonContainer.getEntities().get(0).featureId()); + assertEquals("id2", geojsonContainer.getEntities().get(1).featureId()); + } + + @Test + public void testBrokenJson() { + var container = createLoader("This is a broken json"); + assertEquals( + "Parsing the Geojson file should fail, returning an empty list of entities", + 0, + container.entityCount()); + } + + private GtfsEntityContainer createLoader(String jsonData) { + GeojsonFileLoader loader = new GeojsonFileLoader(); + var fileDescriptor = new GtfsGeojsonFileDescriptor(); + NoticeContainer noticeContainer = new NoticeContainer(); + InputStream inputStream = new ByteArrayInputStream(jsonData.getBytes(StandardCharsets.UTF_8)); + return loader.load(fileDescriptor, null, inputStream, noticeContainer); + } +} From 361d4445ebe8ab97a6c4f82bd3abca72093209b8 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Mon, 30 Sep 2024 22:12:03 -0400 Subject: [PATCH 27/28] Remove old file with uppercase 'J' from index --- .../table/GeoJsonFileLoader.java | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java deleted file mode 100644 index ef532e2007..0000000000 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeoJsonFileLoader.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.mobilitydata.gtfsvalidator.table; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; -import org.mobilitydata.gtfsvalidator.notice.IOError; -import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; -import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; - -/** This class knows how to load a geojson file. */ -public class GeoJsonFileLoader extends TableLoader { - - @Override - public GtfsEntityContainer load( - GtfsFileDescriptor fileDescriptor, - ValidatorProvider validatorProvider, - InputStream inputStream, - NoticeContainer noticeContainer) { - GtfsGeojsonFileDescriptor geojsonFileDescriptor = (GtfsGeojsonFileDescriptor) fileDescriptor; - try { - List entities = extractFeaturesFromStream(inputStream, noticeContainer); - return geojsonFileDescriptor.createContainerForEntities(entities, noticeContainer); - } catch (IOException ioex) { - noticeContainer.addSystemError(new IOError(ioex)); - return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); - } - } - - public List extractFeaturesFromStream( - InputStream inputStream, NoticeContainer noticeContainer) throws IOException { - List features = new ArrayList<>(); - try (InputStreamReader reader = new InputStreamReader(inputStream)) { - JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); - JsonArray featuresArray = jsonObject.getAsJsonArray("features"); - for (JsonElement feature : featuresArray) { - GtfsGeojsonFeature gtfsGeojsonFeature = extractFeature(feature, noticeContainer); - if (gtfsGeojsonFeature != null) { - features.add(gtfsGeojsonFeature); - } - } - } - return features; - } - - public GtfsGeojsonFeature extractFeature(JsonElement feature, NoticeContainer noticeContainer) { - GtfsGeojsonFeature gtfsGeojsonFeature = null; - if (feature.isJsonObject()) { - JsonObject featureObject = feature.getAsJsonObject(); - if (featureObject.has("properties")) { - JsonObject properties = featureObject.getAsJsonObject("properties"); - // Add stop_name and stop_desc - } else { - // Add a notice because properties is required - } - if (featureObject.has("id")) { - gtfsGeojsonFeature = new GtfsGeojsonFeature(); - gtfsGeojsonFeature.setFeatureId(featureObject.get("id").getAsString()); - } else { - // Add a notice because id is required - } - - if (featureObject.has("geometry")) { - JsonObject geometry = featureObject.getAsJsonObject("geometry"); - if (geometry.has("type")) { - String type = geometry.get("type").getAsString(); - if (type.equals("Polygon")) { - // Extract the polygon - } else if (type.equals("Multipolygon")) { - // extract the multipolygon - } - } else { - // Add a notice because type is required - } - } else { - // Add a notice because geometry is required - } - } - return gtfsGeojsonFeature; - } -} From 92a9952b30bd1c221e78e105bee49855770ebec4 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 2 Oct 2024 11:18:58 -0400 Subject: [PATCH 28/28] Added some comments --- .../notice/schema/NoticeSchemaGenerator.java | 1 + .../mobilitydata/gtfsvalidator/table/TableLoader.java | 10 ++++++++++ .../gtfsvalidator/table/GeojsonFileLoader.java | 7 ++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java index 63392bacd2..621fccaa58 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java @@ -127,6 +127,7 @@ private static ReferencesSchema generateReferences(GtfsValidationNotice noticeAn ReferencesSchema schema = new ReferencesSchema(); Arrays.stream(noticeAnnotation.files().value()) .map( + // Both Table and Json annotations specify a file name, collect them all. fileClass -> { Optional fileId = getFileIdForTableClass(fileClass); return fileId.or(() -> getFileIdForJsonClass(fileClass)); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java index 6ba19c3666..c02f54f46e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java @@ -12,6 +12,7 @@ import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; +/** Parent class for the different file loaders. */ public abstract class TableLoader { private static final List> @@ -29,6 +30,15 @@ public static List> getValidatorsWithParsingError return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors); } + /** + * Load the file + * + * @param fileDescriptor Description of the file + * @param validatorProvider Will provide validators to run on the file. + * @param csvInputStream Stream to load from + * @param noticeContainer Where to put the notices if errors occur during the loading. + * @return A container for the loaded entities + */ abstract GtfsEntityContainer load( GtfsFileDescriptor fileDescriptor, ValidatorProvider validatorProvider, diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java index 472e4e64c5..87357ba0f8 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java @@ -15,7 +15,12 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; -/** This class knows how to load a geojson file. */ +/** + * This class knows how to load a geojson file. Typical geojson file: { "type": "FeatureCollection", + * "features": [ { "id": "area_548", "type": "Feature", "geometry": { "type": "Polygon", + * "coordinates": [ [ [ -122.4112929, 48.0834848 ], ... ] ] }, "properties": { "stop_name": "Some + * name", "stop_desc": "Some description" } }, ... ] } + */ public class GeojsonFileLoader extends TableLoader { private static final FluentLogger logger = FluentLogger.forEnclosingClass();