From a85bc6476ba093de6df38bbe5f6f28fc7d162228 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 23 Jun 2023 15:23:09 +0100 Subject: [PATCH 1/9] feat(Bumped GTFS lib version and put a temp fix in place for multi table refs): --- pom.xml | 2 +- .../datatools/manager/jobs/feedmerge/MergeLineContext.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 33a52e498..556e81d9c 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - a3e5707 + 9ea7d34 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index bbcdcbb9c..6793575ea 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -240,7 +240,9 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(field.referenceTable, fieldContext.getValue()); + // FIXME: GTFS-lib is now able to define multiple table references! This is only expecting one, so this update + // will definitely break this part of the code. + String key = getTableScopedValue(field.referenceTables.iterator().next(), fieldContext.getValue()); // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); From 07a455aeedd637cec3be7768001e5a9a157b0c8c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 23 Jun 2023 15:33:42 +0100 Subject: [PATCH 2/9] refactor(MergeLineContext): Updated to use multi table refs --- .../jobs/feedmerge/MergeLineContext.java | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 6793575ea..eefe7981e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -240,38 +240,39 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - // FIXME: GTFS-lib is now able to define multiple table references! This is only expecting one, so this update - // will definitely break this part of the code. - String key = getTableScopedValue(field.referenceTables.iterator().next(), fieldContext.getValue()); - // Check if we're performing a service period merge, this ref field is a service_id, and it - // is not found in the list of service_ids (e.g., it was removed). - boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); - - // If the current foreign ref points to another record that has - // been skipped or is a ref to a non-existent service_id during a service period merge, skip - // this record and add its primary key to the list of skipped IDs (so that other references - // can be properly omitted). - if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { - // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were - // valid service_ids found in calendar_dates, do not skip that record for both the - // calendar_date and any related trips. - if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { - LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); - } else { - String skippedKey = getTableScopedValue(keyValue); - if (orderField != null) { - skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); + // FIXME: GTFS-lib is now able to define multiple table references! This update will most likely break this part of the code. + for (Table referenceTable : field.referenceTables) { + String key = getTableScopedValue(referenceTable, fieldContext.getValue()); + // Check if we're performing a service period merge, this ref field is a service_id, and it + // is not found in the list of service_ids (e.g., it was removed). + boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); + + // If the current foreign ref points to another record that has + // been skipped or is a ref to a non-existent service_id during a service period merge, skip + // this record and add its primary key to the list of skipped IDs (so that other references + // can be properly omitted). + if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { + // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were + // valid service_ids found in calendar_dates, do not skip that record for both the + // calendar_date and any related trips. + if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { + LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); + } else { + String skippedKey = getTableScopedValue(keyValue); + if (orderField != null) { + skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); + } + mergeFeedsResult.skippedIds.add(skippedKey); + return false; } - mergeFeedsResult.skippedIds.add(skippedKey); - return false; } - } - // If the field is a foreign reference, check to see whether the reference has been - // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). - if (mergeFeedsResult.remappedIds.containsKey(key)) { - mergeFeedsResult.remappedReferences++; - // If the value has been remapped update the value to write. - fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); + // If the field is a foreign reference, check to see whether the reference has been + // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). + if (mergeFeedsResult.remappedIds.containsKey(key)) { + mergeFeedsResult.remappedReferences++; + // If the value has been remapped update the value to write. + fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); + } } } return true; From 05f295f1a6d4a43ab906f19072b639256b91036d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 30 Jun 2023 12:18:54 +0100 Subject: [PATCH 3/9] refactor(Merge line context update): Refactored to work with multiple table references --- pom.xml | 2 +- .../CalendarDatesMergeLineContext.java | 1 + .../feedmerge/CalendarMergeLineContext.java | 1 + .../jobs/feedmerge/MergeFeedsResult.java | 14 +- .../jobs/feedmerge/MergeLineContext.java | 122 +++++++++++++----- 5 files changed, 106 insertions(+), 34 deletions(-) diff --git a/pom.xml b/pom.xml index 556e81d9c..55c59adc8 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 9ea7d34 + ba747df diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 649e0c088..d65548e23 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -87,6 +87,7 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept // the valid date range, i.e., before the future feed's first date. if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) { mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + mergeFeedsResult.calendarDatesServiceIds.add(fieldContext.getValueToWrite()); } return !shouldSkipRecord; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index ab8902531..1f6cd27aa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -111,6 +111,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC // If service is going to be cloned, add to the output service ids. if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) { mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + mergeFeedsResult.calendarServiceIds.add(fieldContext.getValueToWrite()); } return !shouldSkipRecord; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index 1adadbb91..5fa9b25b3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -21,10 +21,20 @@ public class MergeFeedsResult implements Serializable { /** Contains the set of IDs for records that were excluded in the merged feed */ public Set skippedIds = new HashSet<>(); /** - * Track the set of service IDs to end up in the merged feed in order to determine which calendar_dates and trips - * records should be retained in the merged result. + * Track the set of service IDs to end up in the merged feed in order to determine which calendar, calendar_dates and + * trip records should be retained in the merged result. */ public Set serviceIds = new HashSet<>(); + + /** + * Track the set of service IDs obtained from calendar records. + */ + public Set calendarServiceIds = new HashSet<>(); + /** + * Track the set of service IDs obtained from calendar date records. + */ + public Set calendarDatesServiceIds = new HashSet<>(); + /** * Track the set of route IDs to end up in the merged feed in order to determine which route_attributes * records should be retained in the merged result. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index eefe7981e..491b4f740 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -237,42 +237,102 @@ public void startNewRow() throws IOException { .collect(Collectors.toList()); } + public enum ReferenceTableLookup { + TRIP_SERVICE_ID_KEY( + String.join(":", Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) + ); + private final String value; + ReferenceTableLookup(String value) { + this.value = value; + } + public String getValue() { + return value; + } + + public static ReferenceTableLookup fromValue(String key) { + for (ReferenceTableLookup ref: ReferenceTableLookup.values()) { + if (ref.getValue().equals(key)) { + return ref; + } + } + throw new UnsupportedOperationException(String.format("Unsupported reference table combination: %s.", key)); + } + } + + /** + * Determine which reference table to use. If there is only one reference use this. If there are multiple references + * determine the context and then the correct reference table to use. + */ + private Table getReferenceTable(FieldContext fieldContext, Field field) { + if (field.referenceTables.size() == 1) { + return field.referenceTables.iterator().next(); + } + Table defaultTable = null; + switch (ReferenceTableLookup.fromValue(createKey(field))) { + case TRIP_SERVICE_ID_KEY: + boolean isCalendarServiceId = mergeFeedsResult.calendarServiceIds.contains(fieldContext.getValueToWrite()); + boolean isCalendarDatesServiceId = mergeFeedsResult.calendarDatesServiceIds.contains(fieldContext.getValueToWrite()); + if (isCalendarServiceId && !isCalendarDatesServiceId) { + return Table.CALENDAR; + } else if (!isCalendarServiceId && isCalendarDatesServiceId) { + return Table.CALENDAR_DATES; + } else { + // A table is still required if the service id is not present in either look-up. For this case it + // doesn't seem to matter which is returned so going with calendar. + defaultTable = Table.CALENDAR; + } + // Add other cases as multiple references are added e.g. flex!. + } + return defaultTable; + } + + /** + * Create a unique key for this table, field and reference tables. + */ + private String createKey(Field field) { + return String.format( + "%s:%s:%s", + table.name, + field.name, + field.referenceTables.stream().map(r -> r.name).collect(Collectors.joining(":")) + ); + } + + + public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - // FIXME: GTFS-lib is now able to define multiple table references! This update will most likely break this part of the code. - for (Table referenceTable : field.referenceTables) { - String key = getTableScopedValue(referenceTable, fieldContext.getValue()); - // Check if we're performing a service period merge, this ref field is a service_id, and it - // is not found in the list of service_ids (e.g., it was removed). - boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); - - // If the current foreign ref points to another record that has - // been skipped or is a ref to a non-existent service_id during a service period merge, skip - // this record and add its primary key to the list of skipped IDs (so that other references - // can be properly omitted). - if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { - // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were - // valid service_ids found in calendar_dates, do not skip that record for both the - // calendar_date and any related trips. - if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { - LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); - } else { - String skippedKey = getTableScopedValue(keyValue); - if (orderField != null) { - skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); - } - mergeFeedsResult.skippedIds.add(skippedKey); - return false; + String key = getTableScopedValue(getReferenceTable(fieldContext, field), fieldContext.getValue()); + // Check if we're performing a service period merge, this ref field is a service_id, and it + // is not found in the list of service_ids (e.g., it was removed). + boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); + + // If the current foreign ref points to another record that has + // been skipped or is a ref to a non-existent service_id during a service period merge, skip + // this record and add its primary key to the list of skipped IDs (so that other references + // can be properly omitted). + if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { + // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were + // valid service_ids found in calendar_dates, do not skip that record for both the + // calendar_date and any related trips. + if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { + LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); + } else { + String skippedKey = getTableScopedValue(keyValue); + if (orderField != null) { + skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); } + mergeFeedsResult.skippedIds.add(skippedKey); + return false; } - // If the field is a foreign reference, check to see whether the reference has been - // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). - if (mergeFeedsResult.remappedIds.containsKey(key)) { - mergeFeedsResult.remappedReferences++; - // If the value has been remapped update the value to write. - fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); - } + } + // If the field is a foreign reference, check to see whether the reference has been + // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). + if (mergeFeedsResult.remappedIds.containsKey(key)) { + mergeFeedsResult.remappedReferences++; + // If the value has been remapped update the value to write. + fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); } } return true; From 38fa3a21e0ba4f270f9ed240d20b0e3bc0a3c556 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 30 Jun 2023 13:00:32 +0100 Subject: [PATCH 4/9] refactor(Addressed line spacing and updated comments): --- .../manager/jobs/feedmerge/MergeFeedsResult.java | 1 + .../manager/jobs/feedmerge/MergeLineContext.java | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index 5fa9b25b3..881c78051 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -30,6 +30,7 @@ public class MergeFeedsResult implements Serializable { * Track the set of service IDs obtained from calendar records. */ public Set calendarServiceIds = new HashSet<>(); + /** * Track the set of service IDs obtained from calendar date records. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 491b4f740..95f6f5420 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -241,10 +241,13 @@ public enum ReferenceTableLookup { TRIP_SERVICE_ID_KEY( String.join(":", Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) ); + private final String value; + ReferenceTableLookup(String value) { this.value = value; } + public String getValue() { return value; } @@ -267,6 +270,8 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { if (field.referenceTables.size() == 1) { return field.referenceTables.iterator().next(); } + + // A table reference is still required even if no match is found. Table defaultTable = null; switch (ReferenceTableLookup.fromValue(createKey(field))) { case TRIP_SERVICE_ID_KEY: @@ -277,8 +282,7 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { } else if (!isCalendarServiceId && isCalendarDatesServiceId) { return Table.CALENDAR_DATES; } else { - // A table is still required if the service id is not present in either look-up. For this case it - // doesn't seem to matter which is returned so going with calendar. + // For this case it doesn't seem to matter which is returned so going with calendar. defaultTable = Table.CALENDAR; } // Add other cases as multiple references are added e.g. flex!. @@ -298,8 +302,6 @@ private String createKey(Field field) { ); } - - public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { From 988fd5153d248b1d746aaa2285c5eadf77149347 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 7 Jul 2023 15:41:30 +0100 Subject: [PATCH 5/9] refactor(Multiple reference tables lookup): Feed merge updated to handle multi ref tables --- pom.xml | 2 +- .../jobs/feedmerge/MergeLineContext.java | 67 ++++----------- .../feedmerge/ReferenceTableDiscovery.java | 83 +++++++++++++++++++ .../manager/jobs/MergeFeedsJobTest.java | 8 +- 4 files changed, 105 insertions(+), 55 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java diff --git a/pom.xml b/pom.xml index 55c59adc8..34eb973c8 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - ba747df + 59b568a diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 95f6f5420..3f6701079 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -237,31 +237,6 @@ public void startNewRow() throws IOException { .collect(Collectors.toList()); } - public enum ReferenceTableLookup { - TRIP_SERVICE_ID_KEY( - String.join(":", Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) - ); - - private final String value; - - ReferenceTableLookup(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - public static ReferenceTableLookup fromValue(String key) { - for (ReferenceTableLookup ref: ReferenceTableLookup.values()) { - if (ref.getValue().equals(key)) { - return ref; - } - } - throw new UnsupportedOperationException(String.format("Unsupported reference table combination: %s.", key)); - } - } - /** * Determine which reference table to use. If there is only one reference use this. If there are multiple references * determine the context and then the correct reference table to use. @@ -271,41 +246,27 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { return field.referenceTables.iterator().next(); } - // A table reference is still required even if no match is found. - Table defaultTable = null; - switch (ReferenceTableLookup.fromValue(createKey(field))) { + switch (ReferenceTableDiscovery.getReferenceTableKey(field, table)) { case TRIP_SERVICE_ID_KEY: - boolean isCalendarServiceId = mergeFeedsResult.calendarServiceIds.contains(fieldContext.getValueToWrite()); - boolean isCalendarDatesServiceId = mergeFeedsResult.calendarDatesServiceIds.contains(fieldContext.getValueToWrite()); - if (isCalendarServiceId && !isCalendarDatesServiceId) { - return Table.CALENDAR; - } else if (!isCalendarServiceId && isCalendarDatesServiceId) { - return Table.CALENDAR_DATES; - } else { - // For this case it doesn't seem to matter which is returned so going with calendar. - defaultTable = Table.CALENDAR; - } - // Add other cases as multiple references are added e.g. flex!. + return ReferenceTableDiscovery.getTripServiceIdReferenceTable( + fieldContext, + mergeFeedsResult, + getTableScopedValue(Table.CALENDAR, fieldContext.getValue()), + getTableScopedValue(Table.CALENDAR_DATES, fieldContext.getValue()) + ); + // Include other cases as multiple references are added e.g. flex!. + default: + return null; } - return defaultTable; - } - - /** - * Create a unique key for this table, field and reference tables. - */ - private String createKey(Field field) { - return String.format( - "%s:%s:%s", - table.name, - field.name, - field.referenceTables.stream().map(r -> r.name).collect(Collectors.joining(":")) - ); } public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(getReferenceTable(fieldContext, field), fieldContext.getValue()); + Table refTable = getReferenceTable(fieldContext, field); + String key = (refTable != null) + ? getTableScopedValue(refTable, fieldContext.getValue()) + : "unknown"; // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java new file mode 100644 index 000000000..9f08d18a3 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java @@ -0,0 +1,83 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.Table; + +import java.util.stream.Collectors; + +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeLineContext.SERVICE_ID; + +public class ReferenceTableDiscovery { + + public static final String REF_TABLE_SEPARATOR = "#~#"; + + public enum ReferenceTableKey { + + TRIP_SERVICE_ID_KEY( + String.join(REF_TABLE_SEPARATOR, Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) + ); + + private final String value; + + ReferenceTableKey(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static ReferenceTableKey fromValue(String key) { + for (ReferenceTableKey ref: ReferenceTableKey.values()) { + if (ref.getValue().equals(key)) { + return ref; + } + } + throw new UnsupportedOperationException(String.format("Unsupported reference table key: %s.", key)); + } + } + + /** + * Get reference table key by matching the provided values to predefined reference table keys. + */ + public static ReferenceTableKey getReferenceTableKey(Field field, Table table) { + return ReferenceTableKey.fromValue(createKey(field, table)); + } + + /** + * Create a unique key for this table, field and reference tables. + */ + public static String createKey(Field field, Table table) { + return String.format( + "%s%s%s%s%s", + table.name, + REF_TABLE_SEPARATOR, + field.name, + REF_TABLE_SEPARATOR, + field.referenceTables.stream().map(r -> r.name).collect(Collectors.joining(REF_TABLE_SEPARATOR)) + ); + } + + /** + * Define the reference table for a trip service id. + */ + public static Table getTripServiceIdReferenceTable( + FieldContext fieldContext, + MergeFeedsResult mergeFeedsResult, + String calendarKey, + String calendarDatesKey + ) { + if ( + mergeFeedsResult.calendarServiceIds.contains(fieldContext.getValueToWrite()) || + mergeFeedsResult.skippedIds.contains(calendarKey) + ) { + return Table.CALENDAR; + } else if ( + mergeFeedsResult.calendarDatesServiceIds.contains(fieldContext.getValueToWrite()) || + mergeFeedsResult.skippedIds.contains(calendarDatesKey) + ) { + return Table.CALENDAR_DATES; + } + return null; + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 430adbfcc..1670f5c3d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -765,7 +765,7 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); - // FIXME: "version3" contains ref integrity errors... was hat intentional? + // FIXME: "version3" contains ref integrity errors... was that intentional? // sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 3 records. @@ -775,12 +775,18 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { // all records from future feed and keep_one from the active feed. sqlAssert.calendarDates.assertCount(3); + // Calendar dates service exception should still be available. + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency5:keep_one'"); + // - trips table should have 3 records. sqlAssert.trips.assertCount(3); // common_id service_id should be scoped for earlier feed version. sqlAssert.trips.assertCount(1, "service_id='Fake_Agency5:common_id'"); + // service_id should still reference calendar dates service exception. + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency5:keep_one'"); + // Amended calendar record from earlier feed version should also have a modified end date (one day before the // earliest start_date from the future feed). sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency5:common_id' AND end_date='20170914'"); From f7f904d46790de4b4cc4fbd2dfe08466404be4ed Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 21 Aug 2023 09:44:51 +0100 Subject: [PATCH 6/9] refactor(pom.xml): Bumped gtfs lib to latest commit for exception based scheduling --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b4b2c27ad..1cd5aba21 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 59b568a + 49a722e161a52cae98fba9cb23995dae82fd01e4 From ffff51a9b38e5a0053a0ea44187c696643b1d440 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 14 Nov 2023 15:32:09 +0000 Subject: [PATCH 7/9] refactor(pom.xml): Bumped GTFS-Lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4de555203..84807c320 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 49a722e161a52cae98fba9cb23995dae82fd01e4 + 9bc752d3ef28e0ea2eda439ebd915fe02a37b0f4 From 0a7763fc28d01459e00dfe0053fb93e79baf437b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 15:08:33 -0500 Subject: [PATCH 8/9] refactor(deps): update gtfs-lib commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b2f2cd2fe..bec80e21d 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.conveyal gtfs-lib - 9bc752d3ef28e0ea2eda439ebd915fe02a37b0f4 + 9bc752d3ef From 811453704cd40370fb6d99321ee83b923424574a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 16 Nov 2023 10:50:30 +0000 Subject: [PATCH 9/9] refactor(ReferenceTableDiscovery.java): Updated trip service id key key now includes reference to schedule exceptions matching trip service_id references --- .../manager/jobs/feedmerge/ReferenceTableDiscovery.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java index df16c2d6f..a972dbc25 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java @@ -14,7 +14,14 @@ public class ReferenceTableDiscovery { public enum ReferenceTableKey { TRIP_SERVICE_ID_KEY( - String.join(REF_TABLE_SEPARATOR, Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) + String.join( + REF_TABLE_SEPARATOR, + Table.TRIPS.name, + SERVICE_ID, + Table.CALENDAR.name, + Table.CALENDAR_DATES.name, + Table.SCHEDULE_EXCEPTIONS.name + ) ); private final String value;