Refactor modules and add data stores (#903)

* Break dependencies between data store and data collection packages * Move the data store package in a dedicated module * Move format specific data store in corresponding modules * Add a data table for openstreetmap data * Add a module and data store for csv data * Add a module and data store for rpsl data * Add a module for the shapefile format and data store * Add a module for the postgres data store * Clean dependencies and improve their scopes
apache · Nov 9, 2024 · 1eeade4 · 1eeade4
1 parent de8f562
commit 1eeade4
Show file tree

Hide file tree

Showing 193 changed files with 1,799 additions and 901 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,6 +51,7 @@ examples/transformation/*.pbf
 
 # Benchmarking
 baremaps-benchmarking/data/
+baremaps-benchmarking/dependency-reduced-pom.xml
 
 # Docs
 .jekyll-cache/

diff --git a/baremaps-benchmarking/pom.xml b/baremaps-benchmarking/pom.xml
@@ -22,6 +22,7 @@
     <dependency>
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-testing</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.openjdk.jmh</groupId>

diff --git a/baremaps-cli/pom.xml b/baremaps-cli/pom.xml
@@ -53,20 +53,9 @@ limitations under the License.
       <artifactId>baremaps-server</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-jul</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-slf4j2-impl</artifactId>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-testing</artifactId>
+      <scope>test</scope>
     </dependency>
   </dependencies>
 

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ExecuteSql.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ExecuteSql.java
@@ -50,7 +50,7 @@ public class ExecuteSql implements Callable<Integer> {
 
   @Override
   public Integer call() throws Exception {
-    new org.apache.baremaps.workflow.tasks.ExecuteSql(database, file.toAbsolutePath(),
+    new org.apache.baremaps.tasks.ExecuteSql(database, file.toAbsolutePath(),
         parallel).execute(new WorkflowContext());
     return 0;
   }

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsm.java
@@ -22,6 +22,7 @@
 import java.nio.file.Path;
 import java.util.concurrent.Callable;
 import org.apache.baremaps.cli.Options;
+import org.apache.baremaps.tasks.ImportOsmPbf;
 import org.apache.baremaps.workflow.WorkflowContext;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Mixin;
@@ -50,7 +51,7 @@ public class ImportOsm implements Callable<Integer> {
 
   @Override
   public Integer call() throws Exception {
-    new org.apache.baremaps.workflow.tasks.ImportOsmPbf(
+    new ImportOsmPbf(
         file.toAbsolutePath(),
         database,
         srid,

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java
@@ -21,8 +21,8 @@
 
 import java.util.concurrent.Callable;
 import org.apache.baremaps.cli.Options;
+import org.apache.baremaps.tasks.UpdateOsmDatabase;
 import org.apache.baremaps.workflow.WorkflowContext;
-import org.apache.baremaps.workflow.tasks.UpdateOsmDatabase;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Mixin;
 import picocli.CommandLine.Option;

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/VectorTileContours.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/VectorTileContours.java
@@ -26,13 +26,13 @@
 import org.apache.baremaps.maplibre.tileset.TilesetLayer;
 import org.apache.baremaps.openstreetmap.stream.ProgressLogger;
 import org.apache.baremaps.openstreetmap.stream.StreamUtils;
+import org.apache.baremaps.tasks.ExportVectorTiles;
 import org.apache.baremaps.tilestore.TileCoord;
 import org.apache.baremaps.tilestore.TileEntry;
 import org.apache.baremaps.tilestore.TileStoreException;
 import org.apache.baremaps.tilestore.pmtiles.PMTilesStore;
 import org.apache.baremaps.tilestore.raster.*;
 import org.apache.baremaps.workflow.WorkflowException;
-import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
 import org.locationtech.jts.geom.Envelope;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -36,13 +36,13 @@
 import org.apache.baremaps.config.ConfigReader;
 import org.apache.baremaps.maplibre.style.Style;
 import org.apache.baremaps.maplibre.tileset.Tileset;
+import org.apache.baremaps.postgres.utils.PostgresUtils;
 import org.apache.baremaps.server.ChangeResource;
 import org.apache.baremaps.server.StyleResource;
 import org.apache.baremaps.server.TilesetResource;
 import org.apache.baremaps.server.VectorTileResource;
 import org.apache.baremaps.tilestore.TileStore;
 import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
-import org.apache.baremaps.utils.PostgresUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Export.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Export.java
@@ -22,8 +22,8 @@
 import java.nio.file.Path;
 import java.util.concurrent.Callable;
 import org.apache.baremaps.cli.Options;
+import org.apache.baremaps.tasks.ExportVectorTiles;
 import org.apache.baremaps.workflow.WorkflowContext;
-import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
@@ -37,14 +37,14 @@
 import org.apache.baremaps.maplibre.style.Style;
 import org.apache.baremaps.maplibre.tilejson.TileJSON;
 import org.apache.baremaps.maplibre.tileset.Tileset;
+import org.apache.baremaps.postgres.utils.PostgresUtils;
 import org.apache.baremaps.server.SearchResource;
 import org.apache.baremaps.server.StyleResource;
 import org.apache.baremaps.server.TileJSONResource;
 import org.apache.baremaps.server.VectorTileResource;
 import org.apache.baremaps.tilestore.TileStore;
 import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
 import org.apache.baremaps.tilestore.vector.VectorTileCache;
-import org.apache.baremaps.utils.PostgresUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;

diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/workflow/Init.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/workflow/Init.java
@@ -23,9 +23,9 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.concurrent.Callable;
+import org.apache.baremaps.tasks.LogMessage;
 import org.apache.baremaps.workflow.Step;
 import org.apache.baremaps.workflow.Workflow;
-import org.apache.baremaps.workflow.tasks.LogMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;

diff --git a/baremaps-core/pom.xml b/baremaps-core/pom.xml
@@ -30,34 +30,10 @@ limitations under the License.
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.dataformat</groupId>
-      <artifactId>jackson-dataformat-csv</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.dataformat</groupId>
-      <artifactId>jackson-dataformat-yaml</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.github.ben-manes.caffeine</groupId>
       <artifactId>caffeine</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
@@ -66,21 +42,13 @@ limitations under the License.
       <groupId>com.zaxxer</groupId>
       <artifactId>HikariCP</artifactId>
     </dependency>
-    <dependency>
-      <groupId>de.bytefish</groupId>
-      <artifactId>pgbulkinsert</artifactId>
-    </dependency>
     <dependency>
       <groupId>it.unimi.dsi</groupId>
       <artifactId>fastutil</artifactId>
     </dependency>
     <dependency>
-      <groupId>mil.nga.geopackage</groupId>
-      <artifactId>geopackage</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>net.ripe.ipresource</groupId>
-      <artifactId>ipresource</artifactId>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-csv</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.baremaps</groupId>
@@ -94,6 +62,10 @@ limitations under the License.
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-flatgeobuf</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-geopackage</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-geoparquet</artifactId>
@@ -110,9 +82,22 @@ limitations under the License.
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-pmtiles</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-postgres</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-rpsl</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-shapefile</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-testing</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -162,10 +147,6 @@ limitations under the License.
       <groupId>org.locationtech.proj4j</groupId>
       <artifactId>proj4j</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.postgresql</groupId>
-      <artifactId>postgresql</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.xerial</groupId>
       <artifactId>sqlite-jdbc</artifactId>

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowConsumer.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowConsumer.java
@@ -18,7 +18,7 @@
 package org.apache.baremaps.geocoder;
 
 import java.util.function.Consumer;
-import org.apache.baremaps.data.storage.DataRow;
+import org.apache.baremaps.store.DataRow;
 import org.apache.lucene.index.IndexWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java
@@ -26,9 +26,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
-import org.apache.baremaps.data.storage.DataColumn;
-import org.apache.baremaps.data.storage.DataRow;
-import org.apache.baremaps.data.storage.DataSchema;
+import org.apache.baremaps.store.DataColumn;
+import org.apache.baremaps.store.DataRow;
+import org.apache.baremaps.store.DataSchema;
 import org.apache.lucene.document.*;
 import org.locationtech.jts.geom.*;
 import org.slf4j.Logger;

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataTableQueryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataTableQueryBuilder.java
@@ -20,8 +20,8 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
-import org.apache.baremaps.data.storage.DataColumn;
-import org.apache.baremaps.data.storage.DataSchema;
+import org.apache.baremaps.store.DataColumn;
+import org.apache.baremaps.store.DataSchema;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.queryparser.classic.QueryParserBase;
 import org.apache.lucene.queryparser.simple.SimpleQueryParser;