Parse additional TransitLine info from GTFS (#252)

add additional line info to gtfs conversion (agency, type, etc) and add tests

authored-by: Tobias Kohl <[email protected]>

.gitignore

@@ -24,3 +24,5 @@ pt2matsim.iml
 test/gtfs-feed-rewrite/
 
 test/stib-mivb-gtfs/
+
+test/Gtfs2TransitScheduleIT/output/

src/main/java/org/matsim/pt2matsim/examples/Workflow.java

@@ -28,7 +28,7 @@ public static void main(String[] args) {
 
 		// convert schedule
 		String unmappedSchedule = "intermediate/schedule_unmapped.xml.gz";
-		Gtfs2TransitSchedule.run("gtfs", "dayWithMostTrips", osmConfig.getOutputCoordinateSystem(), unmappedSchedule, "output/vehicles.xml.gz");
+		Gtfs2TransitSchedule.run("gtfs", "dayWithMostTrips", osmConfig.getOutputCoordinateSystem(), unmappedSchedule, "output/vehicles.xml.gz", "schedule");
 
 		// setup public transit mapper
 		PublicTransitMappingConfigGroup mapperConfig = PublicTransitMappingConfigGroup.createDefaultConfig();

src/main/java/org/matsim/pt2matsim/gtfs/AdditionalTransitLineInfo.java

@@ -0,0 +1,107 @@
+/* *********************************************************************** *
+ * project: org.matsim.*
+ * *********************************************************************** *
+ *                                                                         *
+ * copyright       : (C) 2024 by the members listed in the COPYING,        *
+ *                   LICENSE and WARRANTY file.                            *
+ * email           : info at matsim dot org                                *
+ *                                                                         *
+ * *********************************************************************** *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *   See also COPYING, LICENSE and WARRANTY file                           *
+ *                                                                         *
+ * *********************************************************************** */
+package org.matsim.pt2matsim.gtfs;
+
+import org.matsim.pt2matsim.gtfs.lib.GtfsDefinitions;
+import org.matsim.pt2matsim.gtfs.lib.Route;
+
+/**
+ * Simple container class for useful but not necessary info extracted from the gtfs schedule
+ * 
+ * @author tkohl (Royal2Flush)
+ */
+public class AdditionalTransitLineInfo {
+	public static final String INFO_COLUMN_ID = "lineId";
+	public static final String INFO_COLUMN_SHORTNAME = "shortName";
+	public static final String INFO_COLUMN_LONGNAME = "longName";
+	public static final String INFO_COLUMN_TYPE = "type";
+	public static final String INFO_COLUMN_DESCRIPTION = "description";
+	public static final String INFO_COLUMN_AGENCY_ID = "agencyId";
+	public static final String INFO_COLUMN_AGENCY_NAME = "agencyName";
+	public static final String INFO_COLUMN_AGENCY_URL = "agencyURL";
+	public static final String INFO_COLUMN_NUM_TRANSIT_ROUTES = "numTransitRoutes";
+	public static final String INFO_COLUMN_NUM_TOTAL_DEPARTURES = "totalDepartures";
+
+	private final String id;
+	private final String shortName;
+	private final String longName;
+
+	private final GtfsDefinitions.RouteType routeType;
+	private final String routeDescription;
+	private final String agencyId;
+	private final String agencyName;
+	private final String agencyURL;
+	private int numberOfTransitRoutes = 0;
+	private int totalNumberOfDepartures = 0;
+
+	AdditionalTransitLineInfo(Route gtfsRoute) {
+		this.id = gtfsRoute.getId();
+		this.shortName = gtfsRoute.getShortName();
+		this.longName = gtfsRoute.getLongName();
+		this.routeType = gtfsRoute.getRouteType();
+		this.routeDescription = gtfsRoute.getDescription();
+		this.agencyId = gtfsRoute.getAgency().getId();
+		this.agencyName = gtfsRoute.getAgency().getAgencyName();
+		this.agencyURL = gtfsRoute.getAgency().getAgencyUrl();
+	}
+
+	void countRoute(int numRouteDepartures) {
+		this.numberOfTransitRoutes++;
+		this.totalNumberOfDepartures += numRouteDepartures;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public String getShortName() {
+		return this.shortName;
+	}
+
+	public String getLongName() {
+		return longName;
+	}
+
+	public GtfsDefinitions.RouteType getRouteType() {
+		return routeType;
+	}
+
+	public String getRouteDescription() {
+		return routeDescription;
+	}
+
+	public String getAgencyId() {
+		return agencyId;
+	}
+
+	public String getAgencyName() {
+		return agencyName;
+	}
+
+	public String getAgencyURL() {
+		return agencyURL;
+	}
+
+	public int getNumberOfTransitRoutes() {
+		return numberOfTransitRoutes;
+	}
+
+	public int getTotalNumberOfDepartures() {
+		return totalNumberOfDepartures;
+	}
+}

src/main/java/org/matsim/pt2matsim/gtfs/GtfsConverter.java

@@ -35,6 +35,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 /**
  * Converts a GTFS feed to a MATSim transit schedule
@@ -56,6 +57,7 @@ public class GtfsConverter {
 
 	protected TransitSchedule transitSchedule;
 	protected Vehicles vehiclesContainer;
+	protected Map<Id<TransitLine>, AdditionalTransitLineInfo> additionalLineInfo = new TreeMap<>();
 
 	protected int noStopTimeTrips;
 	protected int stopPairsWithoutOffset;
@@ -80,6 +82,10 @@ public TransitSchedule getSchedule() {
 	public Vehicles getVehicles() {
 		return this.vehiclesContainer;
 	}
+
+	public Map<Id<TransitLine>, AdditionalTransitLineInfo> getAdditionalLineInfo() {
+		return this.additionalLineInfo;
+	}
 
 	/**
 	 * Converts the loaded gtfs data to the given matsim transit schedule
@@ -184,6 +190,7 @@ protected void createTransitLines(TransitSchedule schedule, LocalDate extractDat
 		for(Route gtfsRoute : this.feed.getRoutes().values()) {
 			// create a MATSim TransitLine for each Route
 			TransitLine newTransitLine = createTransitLine(gtfsRoute);
+			AdditionalTransitLineInfo info = additionalLineInfo.get(newTransitLine.getId());
 			if(newTransitLine != null) {
 				schedule.addTransitLine(newTransitLine);
 
@@ -194,6 +201,7 @@ protected void createTransitLines(TransitSchedule schedule, LocalDate extractDat
 						TransitRoute transitRoute = createTransitRoute(trip, schedule.getFacilities());
 						if(transitRoute != null) {
 							newTransitLine.addRoute(transitRoute);
+							info.countRoute(transitRoute.getDepartures().size());
 						}
 					}
 				}
@@ -215,6 +223,8 @@ protected TransitLine createTransitLine(Route gtfsRoute) {
 		Id<TransitLine> id = createTransitLineId(gtfsRoute);
 		TransitLine line = this.scheduleFactory.createTransitLine(id);
 		line.setName(gtfsRoute.getShortName());
+		AdditionalTransitLineInfo info = new AdditionalTransitLineInfo(gtfsRoute);
+		additionalLineInfo.put(id, info);
 		return line;
 	}
 

src/main/java/org/matsim/pt2matsim/gtfs/GtfsFeed.java

@@ -40,6 +40,8 @@ public interface GtfsFeed {
 	Map<String, Service> getServices();
 
 	Map<String, Trip> getTrips();
+
+	Map<String, Agency> getAgencies();
 
 	Map<Id<RouteShape>, RouteShape> getShapes();
 

src/main/java/org/matsim/pt2matsim/gtfs/GtfsFeedImpl.java

@@ -73,6 +73,7 @@ public class GtfsFeedImpl implements GtfsFeed {
 	protected Set<String> serviceIdsNotInCalendarTxt = new HashSet<>();
 
 	// containers for storing gtfs data
+	protected Map<String, Agency> agencies = new HashMap<>();
 	protected Map<String, Stop> stops = new HashMap<>();
 	protected Map<String, Route> routes = new TreeMap<>();
 	protected Map<String, Service> services = new HashMap<>();
@@ -138,6 +139,11 @@ protected void loadFiles(String inputPath) {
 		this.root = inputPath;
 
 		log.info("Loading GTFS files from " + root);
+		try {
+			loadAgencies();
+		} catch (IOException e) {
+			throw new RuntimeException("File agency.txt not found!");
+		}
 		try {
 			loadStops();
 		} catch (IOException e) {
@@ -181,6 +187,44 @@ protected CSVReader createCSVReader(String path) throws FileNotFoundException {
 		InputStream stream = new BOMInputStream(new FileInputStream(path));
 		return new CSVReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
 	}
+
+	/**
+	 * Reads all agencies and puts them in {@link #agencies}
+	 * <p/>
+	 * <br/><br/>
+	 * agency.txt <i>[https://developers.google.com/transit/gtfs/reference]</i><br/>
+	 * Transit agencies with service represented in this dataset.
+	 * @throws FileNotFoundException 
+	 *
+	 * @throws IOException
+	 */
+	protected void loadAgencies() throws IOException {
+		log.info("Loading agency.txt");
+
+		int l = 1;
+		try {
+			CSVReader reader = createCSVReader(root + GtfsDefinitions.Files.AGENCY.fileName);
+			String[] header = reader.readNext(); // read header
+			Map<String, Integer> col = getIndices(header, GtfsDefinitions.Files.AGENCY.columns, GtfsDefinitions.Files.AGENCY.optionalColumns); // get column numbers for required fields
+
+			String[] line = reader.readNext();
+			while(line != null) {
+				l++;
+				String agencyId = line[col.get(GtfsDefinitions.AGENCY_ID)];
+				AgencyImpl agency = new AgencyImpl(agencyId, line[col.get(GtfsDefinitions.AGENCY_NAME)], line[col.get(GtfsDefinitions.AGENCY_URL)], line[col.get(GtfsDefinitions.AGENCY_TIMEZONE)]);
+				agencies.put(agencyId, agency);
+
+				line = reader.readNext();
+			}
+
+			reader.close();
+		} catch (ArrayIndexOutOfBoundsException e) {
+			throw new RuntimeException("Line " + l + " in agency.txt is empty or malformed.");
+		} catch (CsvValidationException e) {
+			throw new RuntimeException(e);
+		}
+		log.info("...     agency.txt loaded");
+	}
 
 	/**
 	 * Reads all stops and puts them in {@link #stops}
@@ -413,7 +457,12 @@ protected void loadRoutes() throws IOException {
 				String routeId = line[col.get(GtfsDefinitions.ROUTE_ID)];
 				String shortName = line[col.get(GtfsDefinitions.ROUTE_SHORT_NAME)];
 				String longName = line[col.get(GtfsDefinitions.ROUTE_LONG_NAME)];
-				Route newGtfsRoute = new RouteImpl(routeId, shortName, longName, extendedRouteType);
+
+				Agency agency = this.agencies.get(line[col.get(GtfsDefinitions.AGENCY_ID)]);
+				if (agency == null) {
+					throw new RuntimeException("Line " + l + " in routes.txt references unknown agency id " + line[col.get(GtfsDefinitions.AGENCY_ID)]);
+				}
+				Route newGtfsRoute = new RouteImpl(routeId, shortName, longName, agency, extendedRouteType);
 				routes.put(line[col.get(GtfsDefinitions.ROUTE_ID)], newGtfsRoute);
 
 				line = reader.readNext();
@@ -752,5 +801,10 @@ public Map<String, Service> getServices() {
 	public Map<String, Trip> getTrips() {
 		return trips;
 	}
+
+	@Override
+	public Map<String, Agency> getAgencies() {
+		return agencies;
+	}
 
 }

src/main/java/org/matsim/pt2matsim/gtfs/lib/Agency.java

@@ -19,10 +19,11 @@
 package org.matsim.pt2matsim.gtfs.lib;
 
 /**
- * Not implemented
  * @author polettif
  */
 public interface Agency {
+
+	String getId();
 
 	String getAgencyName();
 

src/main/java/org/matsim/pt2matsim/gtfs/lib/AgencyImpl.java

@@ -0,0 +1,84 @@
+/* *********************************************************************** *
+ * project: org.matsim.*
+ * *********************************************************************** *
+ *                                                                         *
+ * copyright       : (C) 2024 by the members listed in the COPYING,        *
+ *                   LICENSE and WARRANTY file.                            *
+ * email           : info at matsim dot org                                *
+ *                                                                         *
+ * *********************************************************************** *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *   See also COPYING, LICENSE and WARRANTY file                           *
+ *                                                                         *
+ * *********************************************************************** */
+package org.matsim.pt2matsim.gtfs.lib;
+
+/**
+ * @author tkohl (Royal2Flush)
+ */
+public class AgencyImpl implements Agency {
+
+	private final String id;
+	private final String name;
+	private final String url;
+	private final String timezone;
+
+	public AgencyImpl(String id, String name, String url, String timezone) {
+		this.id = id;
+		this.name = name;
+		this.url = url;
+		this.timezone = timezone;
+	}
+
+	@Override
+	public String getId() {
+		return this.id;
+	}
+
+	@Override
+	public String getAgencyName() {
+		return this.name;
+	}
+
+	@Override
+	public String getAgencyUrl() {
+		return this.url;
+	}
+
+	@Override
+	public String getAgencyTimeZone() {
+		return this.timezone;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if(this == o) return true;
+		if(o == null || getClass() != o.getClass()) return false;
+
+		AgencyImpl other = (AgencyImpl) o;
+
+		if(!this.id.equals(other.id)) return false;
+		if(!this.name.equals(other.name)) return false;
+		if(!this.url.equals(other.url)) return false;
+		return this.timezone.equals(other.timezone);
+	}
+
+	@Override
+	public int hashCode() {
+		int result = id.hashCode();
+		result = 31 * result + name.hashCode();
+		result = 31 * result + url.hashCode();
+		result = 31 * result + timezone.hashCode();
+		return result;
+	}
+
+	@Override
+	public String toString() {
+		return "[agency:" + id + ", \"" + name + "\"]";
+	}
+
+}