Merge branch 'master' into HDDS-11699

apache · Jan 9, 2025 · b37dd91 · b37dd91
2 parents 3fc4af1 + f1f0ec3
commit b37dd91
Show file tree

Hide file tree

Showing 456 changed files with 4,770 additions and 5,953 deletions.
diff --git a/hadoop-hdds/annotations/pom.xml b/hadoop-hdds/annotations/pom.xml
@@ -12,10 +12,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.ozone</groupId>
@@ -25,14 +22,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <artifactId>hdds-annotation-processing</artifactId>
   <version>2.0.0-SNAPSHOT</version>
-  <description>Apache Ozone annotation processing tools for validating custom
-    annotations at compile time.
-  </description>
-  <name>Apache Ozone Annotation Processing</name>
   <packaging>jar</packaging>
+  <name>Apache Ozone Annotation Processing</name>
+  <description>Apache Ozone annotation processing tools for validating custom
+    annotations at compile time.</description>
 
   <properties>
-    <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
+    <!-- no tests in this module so far -->
+    <maven.test.skip>true</maven.test.skip>
   </properties>
 
   <build>

diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml
@@ -12,10 +12,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.ozone</groupId>
@@ -25,14 +22,31 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <artifactId>hdds-client</artifactId>
   <version>2.0.0-SNAPSHOT</version>
-  <description>Apache Ozone Distributed Data Store Client Library</description>
-  <name>Apache Ozone HDDS Client</name>
   <packaging>jar</packaging>
-
-  <properties>
-  </properties>
+  <name>Apache Ozone HDDS Client</name>
+  <description>Apache Ozone Distributed Data Store Client Library</description>
 
   <dependencies>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentracing</groupId>
+      <artifactId>opentracing-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentracing</groupId>
+      <artifactId>opentracing-util</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.annotation</groupId>
+      <artifactId>jakarta.annotation-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-common</artifactId>
@@ -49,11 +63,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-interface-client</artifactId>
     </dependency>
-
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-client</artifactId>
@@ -74,31 +83,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-thirdparty-misc</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>io.opentracing</groupId>
-      <artifactId>opentracing-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.opentracing</groupId>
-      <artifactId>opentracing-util</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>jakarta.annotation</groupId>
-      <artifactId>jakarta.annotation-api</artifactId>
-    </dependency>
-
     <!-- Test dependencies -->
     <dependency>
       <groupId>org.apache.ozone</groupId>
@@ -148,7 +137,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <artifactId>maven-enforcer-plugin</artifactId>
         <executions>
           <execution>
-            <id>ban-annotations</id> <!-- override default restriction from root POM -->
+            <id>ban-annotations</id>
+            <!-- override default restriction from root POM -->
             <configuration>
               <rules>
                 <restrictImports>

diff --git a/...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java b/...p-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java
@@ -253,7 +253,7 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
     }
     while (len > 0) {
       allocateNewBufferIfNeeded();
-      int writeLen = Math.min(len, currentBuffer.length());
+      int writeLen = Math.min(len, currentBuffer.remaining());
       final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
       currentBuffer.put(buf);
       writeChunkIfNeeded();
@@ -265,7 +265,7 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
   }
 
   private void writeChunkIfNeeded() throws IOException {
-    if (currentBuffer.length() == 0) {
+    if (currentBuffer.remaining() == 0) {
       writeChunk(currentBuffer);
       currentBuffer = null;
     }
@@ -410,6 +410,10 @@ public void executePutBlock(boolean close,
     waitFuturesComplete();
     final BlockData blockData = containerBlockData.build();
     if (close) {
+      // HDDS-12007 changed datanodes to ignore the following PutBlock request.
+      // However, clients still have to send it for maintaining compatibility.
+      // Otherwise, new clients won't send a PutBlock.
+      // Then, old datanodes will fail since they expect a PutBlock.
       final ContainerCommandRequestProto putBlockRequest
           = ContainerProtocolCalls.getPutBlockRequest(
               xceiverClient.getPipeline(), blockData, true, tokenString);
@@ -672,6 +676,7 @@ private void writeChunkToContainer(ByteBuffer buf)
             out.writeAsync(buf, StandardWriteOption.SYNC) :
             out.writeAsync(buf))
             .whenCompleteAsync((r, e) -> {
+              metrics.decrPendingContainerOpsMetrics(ContainerProtos.Type.WriteChunk);
               if (e != null || !r.isSuccess()) {
                 if (e == null) {
                   e = new IOException("result is not success");

diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java
@@ -39,8 +39,8 @@ public ByteBuffer duplicate() {
     return buffer.duplicate();
   }
 
-  public int length() {
-    return buffer.limit() - buffer.position();
+  public int remaining() {
+    return buffer.remaining();
   }
 
   public int position() {