locations.geojson POC phase2: End-to-end partial support of json data

.gitignore

@@ -104,5 +104,6 @@ app/pkg/bin/
 processor/notices/bin/
 processor/notices/tests/bin/
 web/service/bin/
+/web/service/execution_result.json
 
 RULES.md

core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java

@@ -32,6 +32,7 @@
 import java.util.Optional;
 import java.util.TreeMap;
 import java.util.logging.Level;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsJson;
 import org.mobilitydata.gtfsvalidator.annotation.GtfsTable;
 import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;
 import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef;
@@ -125,7 +126,11 @@ public static NoticeDocComments loadComments(Class<?> noticeClass) {
   private static ReferencesSchema generateReferences(GtfsValidationNotice noticeAnnotation) {
     ReferencesSchema schema = new ReferencesSchema();
     Arrays.stream(noticeAnnotation.files().value())
-        .map(NoticeSchemaGenerator::getFileIdForTableClass)
+        .map(
+            fileClass -> {
+              Optional<String> fileId = getFileIdForTableClass(fileClass);
+              return fileId.or(() -> getFileIdForJsonClass(fileClass));
+            })
         .flatMap(Optional::stream)
         .forEach(schema::addFileReference);
     Arrays.stream(noticeAnnotation.bestPractices().value())
@@ -146,6 +151,11 @@ private static Optional<String> getFileIdForTableClass(Class<? extends GtfsEntit
     return Optional.ofNullable(table).map(GtfsTable::value);
   }
 
+  private static Optional<String> getFileIdForJsonClass(Class<? extends GtfsEntity> entityClass) {
+    GtfsJson annotation = entityClass.getAnnotation(GtfsJson.class);
+    return Optional.ofNullable(annotation).map(GtfsJson::value);
+  }
+
   private static UrlReference convertUrlRef(UrlRef ref) {
     return new UrlReference(ref.label(), ref.url());
   }

core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java → core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java

@@ -7,48 +7,44 @@
 import com.univocity.parsers.csv.CsvParserSettings;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.mobilitydata.gtfsvalidator.notice.CsvParsingFailedNotice;
 import org.mobilitydata.gtfsvalidator.notice.EmptyFileNotice;
-import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice;
-import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice;
 import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
 import org.mobilitydata.gtfsvalidator.parsing.CsvFile;
 import org.mobilitydata.gtfsvalidator.parsing.CsvHeader;
 import org.mobilitydata.gtfsvalidator.parsing.CsvRow;
 import org.mobilitydata.gtfsvalidator.parsing.FieldCache;
 import org.mobilitydata.gtfsvalidator.parsing.RowParser;
-import org.mobilitydata.gtfsvalidator.validator.FileValidator;
 import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator;
 import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider;
 import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil;
 
-public final class AnyTableLoader {
+/** This class loads csv files specifically. */
+public final class CsvFileLoader extends TableLoader {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final List<Class<? extends FileValidator>> singleFileValidatorsWithParsingErrors =
-      new ArrayList<>();
+  private CsvFileLoader() {}
+  // Create the singleton and add a method to obtain it
+  private static final CsvFileLoader INSTANCE = new CsvFileLoader();
 
-  private static final List<Class<? extends SingleEntityValidator>>
-      singleEntityValidatorsWithParsingErrors = new ArrayList<>();
-
-  public List<Class<? extends FileValidator>> getValidatorsWithParsingErrors() {
-    return Collections.unmodifiableList(singleFileValidatorsWithParsingErrors);
+  @Nonnull
+  public static CsvFileLoader getInstance() {
+    return INSTANCE;
   }
 
-  public List<Class<? extends SingleEntityValidator>> getSingleEntityValidatorsWithParsingErrors() {
-    return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors);
-  }
+  private final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  public static GtfsTableContainer load(
-      GtfsTableDescriptor tableDescriptor,
+  @Override
+  public GtfsEntityContainer<?, ?> load(
+      GtfsFileDescriptor fileDescriptor,
       ValidatorProvider validatorProvider,
       InputStream csvInputStream,
       NoticeContainer noticeContainer) {
+    GtfsTableDescriptor tableDescriptor = (GtfsTableDescriptor) fileDescriptor;
     final String gtfsFilename = tableDescriptor.gtfsFilename();
 
     CsvFile csvFile;
@@ -61,22 +57,19 @@ public static GtfsTableContainer load(
       csvFile = new CsvFile(csvInputStream, gtfsFilename, settings);
     } catch (TextParsingException e) {
       noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.INVALID_HEADERS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
     }
     if (csvFile.isEmpty()) {
       noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.EMPTY_FILE);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE);
     }
     final CsvHeader header = csvFile.getHeader();
     final ImmutableList<GtfsColumnDescriptor> columnDescriptors = tableDescriptor.getColumns();
     final NoticeContainer headerNotices =
         validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors);
     noticeContainer.addAll(headerNotices);
     if (headerNotices.hasValidationErrors()) {
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.INVALID_HEADERS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
     }
     final int nColumns = columnDescriptors.size();
     final ImmutableMap<String, GtfsFieldLoader> fieldLoadersMap = tableDescriptor.getFieldLoaders();
@@ -99,8 +92,8 @@ public static GtfsTableContainer load(
     final List<GtfsEntity> entities = new ArrayList<>();
     boolean hasUnparsableRows = false;
     final List<SingleEntityValidator<GtfsEntity>> singleEntityValidators =
-        validatorProvider.createSingleEntityValidators(
-            tableDescriptor.getEntityClass(), singleEntityValidatorsWithParsingErrors::add);
+        createSingleEntityValidators(tableDescriptor.getEntityClass(), validatorProvider);
+
     try {
       for (CsvRow row : csvFile) {
         if (row.getRowNumber() % 200000 == 0) {
@@ -133,26 +126,23 @@ public static GtfsTableContainer load(
       }
     } catch (TextParsingException e) {
       noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
     } finally {
       logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors);
     }
     if (hasUnparsableRows) {
       logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename);
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
     }
     GtfsTableContainer table =
         tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer);
+
     ValidatorUtil.invokeSingleFileValidators(
-        validatorProvider.createSingleFileValidators(
-            table, singleFileValidatorsWithParsingErrors::add),
-        noticeContainer);
+        createSingleFileValidators(table, validatorProvider), noticeContainer);
     return table;
   }
 
-  private static NoticeContainer validateHeaders(
+  private NoticeContainer validateHeaders(
       ValidatorProvider validatorProvider,
       String gtfsFilename,
       CsvHeader header,
@@ -178,7 +168,7 @@ private static NoticeContainer validateHeaders(
     return headerNotices;
   }
 
-  private static void logFieldCacheStats(
+  private void logFieldCacheStats(
       String gtfsFilename,
       FieldCache[] fieldCaches,
       ImmutableList<GtfsColumnDescriptor> columnDescriptors) {
@@ -196,25 +186,4 @@ private static void logFieldCacheStats(
       }
     }
   }
-
-  public static GtfsTableContainer loadMissingFile(
-      GtfsTableDescriptor tableDescriptor,
-      ValidatorProvider validatorProvider,
-      NoticeContainer noticeContainer) {
-    String gtfsFilename = tableDescriptor.gtfsFilename();
-    GtfsTableContainer table =
-        tableDescriptor.createContainerForInvalidStatus(
-            GtfsTableContainer.TableStatus.MISSING_FILE);
-    if (tableDescriptor.isRecommended()) {
-      noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename));
-    }
-    if (tableDescriptor.isRequired()) {
-      noticeContainer.addValidationNotice(new MissingRequiredFileNotice(gtfsFilename));
-    }
-    ValidatorUtil.invokeSingleFileValidators(
-        validatorProvider.createSingleFileValidators(
-            table, singleFileValidatorsWithParsingErrors::add),
-        noticeContainer);
-    return table;
-  }
 }

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java

@@ -0,0 +1,58 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * This class is the parent of containers holding table (csv) entities and containers holding JSON
+ * entities
+ *
+ * @param <T> The entity for this container (e.g. GtfsCalendarDate or GtfsGeojsonFeature )
+ * @param <D> The descriptor for the file for the container (e.g. GtfsCalendarDateTableDescriptor or
+ *     GtfsGeojsonFileDescriptor)
+ */
+public abstract class GtfsEntityContainer<T extends GtfsEntity, D extends GtfsFileDescriptor> {
+
+  private final D descriptor;
+  private final TableStatus tableStatus;
+
+  public GtfsEntityContainer(D descriptor, TableStatus tableStatus) {
+    this.tableStatus = tableStatus;
+    this.descriptor = descriptor;
+  }
+
+  public TableStatus getTableStatus() {
+    return tableStatus;
+  }
+
+  public D getDescriptor() {
+    return descriptor;
+  }
+
+  public abstract Class<T> getEntityClass();
+
+  public int entityCount() {
+    return getEntities().size();
+  }
+
+  public abstract List<T> getEntities();
+
+  public abstract String gtfsFilename();
+
+  public abstract Optional<T> byTranslationKey(String recordId, String recordSubId);
+
+  public boolean isMissingFile() {
+    return tableStatus == TableStatus.MISSING_FILE;
+  }
+
+  public boolean isParsedSuccessfully() {
+    switch (tableStatus) {
+      case PARSABLE_HEADERS_AND_ROWS:
+        return true;
+      case MISSING_FILE:
+        return !descriptor.isRequired();
+      default:
+        return false;
+    }
+  }
+}

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java

@@ -18,20 +18,19 @@
 
 import com.google.common.base.Ascii;
 import java.util.*;
-import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
 
 /**
  * Container for a whole parsed GTFS feed with all its tables.
  *
- * <p>The tables are kept as {@code GtfsTableContainer} instances.
+ * <p>The tables are kept as {@link GtfsEntityContainer} instances.
  */
 public class GtfsFeedContainer {
-  private final Map<String, GtfsTableContainer<?>> tables = new HashMap<>();
-  private final Map<Class<? extends GtfsTableContainer>, GtfsTableContainer<?>> tablesByClass =
+  private final Map<String, GtfsEntityContainer<?, ?>> tables = new HashMap<>();
+  private final Map<Class<? extends GtfsEntityContainer>, GtfsEntityContainer<?, ?>> tablesByClass =
       new HashMap<>();
 
-  public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
-    for (GtfsTableContainer<?> table : tableContainerList) {
+  public GtfsFeedContainer(List<GtfsEntityContainer<?, ?>> tableContainerList) {
+    for (GtfsEntityContainer<?, ?> table : tableContainerList) {
       tables.put(table.gtfsFilename(), table);
       tablesByClass.put(table.getClass(), table);
     }
@@ -49,11 +48,12 @@ public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
    * @param filename file name, including ".txt" extension
    * @return GTFS table or empty if the table is not supported by schema
    */
-  public Optional<GtfsTableContainer<?>> getTableForFilename(String filename) {
-    return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
+  public <T extends GtfsEntityContainer<?, ?>> Optional<T> getTableForFilename(String filename) {
+    return (Optional<T>)
+        Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
   }
 
-  public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
+  public <T extends GtfsEntityContainer<?, ?>> T getTable(Class<T> clazz) {
     return (T) tablesByClass.get(clazz);
   }
 
@@ -65,21 +65,21 @@ public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
    * @return true if all files were successfully parsed, false otherwise
    */
   public boolean isParsedSuccessfully() {
-    for (GtfsTableContainer<?> table : tables.values()) {
+    for (GtfsEntityContainer<?, ?> table : tables.values()) {
       if (!table.isParsedSuccessfully()) {
         return false;
       }
     }
     return true;
   }
 
-  public Collection<GtfsTableContainer<?>> getTables() {
+  public Collection<GtfsEntityContainer<?, ?>> getTables() {
     return tables.values();
   }
 
   public String tableTotalsText() {
     List<String> totalList = new ArrayList<>();
-    for (GtfsTableContainer<?> table : tables.values()) {
+    for (GtfsEntityContainer<?, ?> table : tables.values()) {
       if (table.getTableStatus() == TableStatus.MISSING_FILE
           && !table.getDescriptor().isRequired()) {
         continue;