HDDS-11746. Support arbitrary schemas as an option in ozone debug ldb.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
 import org.apache.hadoop.hdds.utils.db.LongCodec;
 import org.apache.hadoop.hdds.utils.CollectionUtils;
@@ -79,6 +80,10 @@ public DatanodeSchemaOneDBDefinition(String dbPath,
     super(dbPath, config);
   }
 
+  public DatanodeSchemaOneDBDefinition(String dbPath) {
+    super(dbPath, new OzoneConfiguration());
+  }
+
   @Override
   public DBColumnFamilyDefinition<String, BlockData>
       getBlockDataColumnFamily() {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java

@@ -19,6 +19,7 @@
 
 import com.google.common.primitives.Longs;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
 import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
 import org.apache.hadoop.hdds.utils.db.DBDefinition;
@@ -124,6 +125,10 @@ public DatanodeSchemaThreeDBDefinition(String dbPath,
     LAST_CHUNK_INFO.setCfOptions(cfOptions);
   }
 
+  public DatanodeSchemaThreeDBDefinition(String dbPath) {
+    super(dbPath, new OzoneConfiguration());
+  }
+
   @Override
   public Map<String, DBColumnFamilyDefinition<?, ?>> getMap() {
     return COLUMN_FAMILIES;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.ozone.container.metadata;
 
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
 import org.apache.hadoop.hdds.utils.db.DBDefinition;
 import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
@@ -79,6 +80,10 @@ public DatanodeSchemaTwoDBDefinition(String dbPath,
     super(dbPath, config);
   }
 
+  public DatanodeSchemaTwoDBDefinition(String dbPath) {
+    super(dbPath, new OzoneConfiguration());
+  }
+
   private static final Map<String, DBColumnFamilyDefinition<?, ?>>
       COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
           BLOCK_DATA,

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java

@@ -138,6 +138,10 @@ protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
     super(map);
   }
 
+  public SCMDBDefinition() {
+    this(COLUMN_FAMILIES);
+  }
+
   @Override
   public String getName() {
     return "scm.db";

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java

@@ -26,9 +26,12 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
 import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
 import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -80,6 +83,8 @@ public class TestLDBCli {
   private static final String KEY_TABLE = "keyTable";
   private static final String BLOCK_DATA = "block_data";
   public static final String PIPELINES = "pipelines";
+  public static final String DUMMY_DB = "anotherKeyTable";
+  public static final String ANOTHER_KEY_TABLE_NAME = "anotherKeyTable";
   private static final ObjectMapper MAPPER = new ObjectMapper();
   private OzoneConfiguration conf;
   private DBStore dbStore;
@@ -410,6 +415,40 @@ void testSchemaCommand() throws IOException {
     assertEquals("", stderr.toString());
   }
 
+  @Test
+  void testCommandsWithDBDefOverride() throws IOException {
+    // Prepare dummy table
+    prepareTable(DUMMY_DB, true);
+
+    // Prepare args for value-schema command
+    List<String> completeScanArgs = new ArrayList<>(Arrays.asList(
+        "--db", dbStore.getDbLocation().getAbsolutePath(),
+        "--schema", DummyDBDefinition.class.getName(),
+        "value-schema",
+        "--column-family", ANOTHER_KEY_TABLE_NAME));
+
+    int exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+    assertEquals(0, exitCode, stderr.toString());
+    Pattern p = Pattern.compile(".*String.*String.*", Pattern.MULTILINE);
+    Matcher m = p.matcher(stdout.toString());
+    assertTrue(m.find());
+    assertEquals("", stderr.toString());
+
+    // Prepare args for scan command
+    completeScanArgs = new ArrayList<>(Arrays.asList(
+        "--db", dbStore.getDbLocation().getAbsolutePath(),
+        "--schema", DummyDBDefinition.class.getName(),
+        "scan",
+        "--column-family", ANOTHER_KEY_TABLE_NAME));
+
+    exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+    assertEquals(0, exitCode, stderr.toString());
+    p = Pattern.compile(".*random-key.*random-value.*", Pattern.MULTILINE);
+    m = p.matcher(stdout.toString());
+    assertTrue(m.find());
+    assertEquals("", stderr.toString());
+  }
+
   /**
    * Converts String input to a Map and compares to the given Map input.
    * @param expected expected result Map
@@ -473,11 +512,23 @@ private void prepareTable(String tableName, boolean schemaV3)
         }
       }
       break;
+
     case PIPELINES:
       // Empty table
       dbStore = DBStoreBuilder.newBuilder(conf).setName("scm.db")
           .setPath(tempDir.toPath()).addTable(PIPELINES).build();
       break;
+
+    case DUMMY_DB:
+      dbStore = DBStoreBuilder.newBuilder(new OzoneConfiguration())
+          .setName("another.db")
+          .setPath(tempDir.toPath())
+          .addTable(ANOTHER_KEY_TABLE_NAME)
+          .build();
+      dbStore.getTable(ANOTHER_KEY_TABLE_NAME, String.class, String.class)
+          .put("random-key", "random-value");
+      break;
+
     default:
       throw new IllegalArgumentException("Unsupported table: " + tableName);
     }
@@ -512,4 +563,25 @@ private static Map<String, Object> toMap(Object obj) throws IOException {
     return MAPPER.readValue(json, new TypeReference<Map<String, Object>>() { });
   }
 
+  /**
+   * New DBDefinition to test arbitrary schemas for ldb commands.
+   */
+  public static class DummyDBDefinition extends DBDefinition.WithMap {
+    public static final DBColumnFamilyDefinition<String, String> ANOTHER_KEY_TABLE
+        = new DBColumnFamilyDefinition<>(ANOTHER_KEY_TABLE_NAME, StringCodec.get(), StringCodec.get());
+    private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+        COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(ANOTHER_KEY_TABLE);
+    protected DummyDBDefinition() {
+      super(COLUMN_FAMILIES);
+    }
+    @Override
+    public String getName() {
+      return "another.db";
+    }
+    @Override
+    public String getLocationConfigKey() {
+      return "";
+    }
+  }
+
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.ozone.debug;
 
+import java.lang.reflect.Constructor;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collections;
@@ -98,6 +99,36 @@ public static DBDefinition getDefinition(Path dbPath,
     return getDefinition(dbName);
   }
 
+  public static DBDefinition getDefinition(Path dbPath, ConfigurationSource config, String overrideDBDef) {
+    if (overrideDBDef == null) {
+      return getDefinition(dbPath, config);
+    }
+    try {
+      Class<?> clazz = Class.forName(overrideDBDef);
+      if (DBDefinition.class.isAssignableFrom(clazz)) {
+        DBDefinition instance;
+        try {
+          Constructor<?> noParamConstructor = clazz.getDeclaredConstructor();
+          noParamConstructor.setAccessible(true);
+          instance = (DBDefinition) noParamConstructor.newInstance();
+        } catch (NoSuchMethodException e) {
+          Constructor<?> stringParamConstructor = clazz.getDeclaredConstructor(String.class);
+          stringParamConstructor.setAccessible(true);
+          instance = (DBDefinition) stringParamConstructor.newInstance(dbPath.toAbsolutePath().toString());
+        }
+        return instance;
+      } else {
+        System.err.println("Class does not implement DBDefinition: " + overrideDBDef);
+      }
+    } catch (ClassNotFoundException e) {
+      System.err.println("Class not found: " + overrideDBDef);
+    } catch (InstantiationException | IllegalAccessException | NoSuchMethodException |
+             SecurityException | java.lang.reflect.InvocationTargetException e) {
+      System.err.println("Error creating instance: " + e.getMessage());
+    }
+    return null;
+  }
+
   private static DBDefinition getReconDBDefinition(String dbName) {
     if (dbName.startsWith(RECON_CONTAINER_KEY_DB)) {
       return new ReconDBDefinition(dbName);

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java

@@ -581,7 +581,7 @@ private boolean printTable(List<ColumnFamilyHandle> columnFamilyHandleList,
     dbPath = removeTrailingSlashIfNeeded(dbPath);
     DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion);
     DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(
-        Paths.get(dbPath), new OzoneConfiguration());
+        Paths.get(dbPath), new OzoneConfiguration(), parent.getDbDefinition());
     if (dbDefinition == null) {
       err().println("Error: Incorrect DB Path");
       return false;

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java

@@ -43,11 +43,19 @@ public class RDBParser implements DebugSubcommand {
       description = "Database File Path")
   private String dbPath;
 
+  @CommandLine.Option(names = {"--schema"},
+      description = "DBDefinition of the database")
+  private String dbDefinition;
+
   public String getDbPath() {
     return dbPath;
   }
 
   public void setDbPath(String dbPath) {
     this.dbPath = dbPath;
   }
+
+  public String getDbDefinition() {
+    return dbDefinition;
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/ValueSchema.java

@@ -86,7 +86,7 @@ public Void call() throws Exception {
 
     String dbPath = parent.getDbPath();
     Map<String, Object> fields = new HashMap<>();
-    success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion);
+    success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion, parent.getDbDefinition());
 
     out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(fields));
 
@@ -100,11 +100,11 @@ public Void call() throws Exception {
   }
 
   public static boolean getValueFields(String dbPath, Map<String, Object> valueSchema, int d, String table,
-                                       String dnDBSchemaVersion) {
+                                       String dnDBSchemaVersion, String dbDef) {
 
     dbPath = removeTrailingSlashIfNeeded(dbPath);
     DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion);
-    DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration());
+    DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration(), dbDef);
     if (dbDefinition == null) {
       err().println("Error: Incorrect DB Path");
       return false;

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java

@@ -18,9 +18,13 @@
 
 package org.apache.hadoop.ozone.debug;
 
+import java.lang.reflect.Constructor;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
 import org.apache.hadoop.hdds.utils.db.DBDefinition;
@@ -34,8 +38,12 @@
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB;
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.junit.jupiter.api.Test;
+import org.reflections.Reflections;
 
 /**
  * Simple factory unit test.
@@ -75,4 +83,46 @@ public void testGetDefinition() {
     definition = DBDefinitionFactory.getDefinition(dbPath, conf);
     assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition);
   }
+
+  @Test
+  public void testGetDefinitionWithOverride() {
+    final OzoneConfiguration conf = new OzoneConfiguration();
+    Path dbPath = Paths.get("another.db");
+    DBDefinition definition = DBDefinitionFactory.getDefinition(dbPath, conf, OMDBDefinition.class.getName());
+    assertInstanceOf(OMDBDefinition.class, definition);
+  }
+
+  /*
+   * Test to ensure that any DBDefinition has a default constructor or a constructor with 1 parameter.
+   * This is needed for ldb tools to run with arbitrary DB definitions.
+   */
+  @Test
+  public void testAllDBDefinitionsHaveCorrectConstructor() {
+    Set<Class<? extends DBDefinition.WithMap>> withMapSubclasses = new HashSet<>();
+    try {
+      Reflections reflections = new Reflections("org.apache.hadoop");
+      withMapSubclasses = reflections.getSubTypesOf(DBDefinition.WithMap.class);
+    } catch (Exception e) {
+      fail("Error while finding subclasses: " + e.getMessage());
+    }
+    assertFalse(withMapSubclasses.isEmpty(), "No classes found extending DBDefinition.WithMap");
+    System.out.println(withMapSubclasses);
+
+    // Check constructors for each subclass
+    for (Class<?> clazz : withMapSubclasses) {
+      Constructor<?>[] constructors = clazz.getDeclaredConstructors();
+      boolean hasValidConstructor = false;
+
+      for (Constructor<?> constructor : constructors) {
+        int paramCount = constructor.getParameterCount();
+        if (paramCount == 0 || paramCount == 1) {
+          hasValidConstructor = true;
+          break;
+        }
+      }
+
+      assertTrue(hasValidConstructor,
+          "Class " + clazz.getName() + " does not have a valid constructor (default or single parameter)");
+    }
+  }
 }