From f5e00f7df337b7fd35b2dfe546a307d5552b01b1 Mon Sep 17 00:00:00 2001 From: Johnny Sohn Date: Thu, 18 Apr 2024 09:27:25 -0700 Subject: [PATCH] only check for delegation token from NN when it is non S3 filesystem --- .../hadoop/mapreduce/lib/output/FileOutputFormat.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java index 15ff3c67c2fb5..d46f0820650e9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java @@ -156,9 +156,12 @@ public void checkOutputSpecs(JobContext job throw new InvalidJobConfException("Output directory not set."); } - // get delegation token for outDir's file system - TokenCache.obtainTokensForNamenodes(job.getCredentials(), - new Path[] { outDir }, job.getConfiguration()); + // only get delegation token for non-S3 file system + if (!outDir.toString().startsWith("s3")) { + // get delegation token for outDir's file system + TokenCache.obtainTokensForNamenodes(job.getCredentials(), + new Path[] { outDir }, job.getConfiguration()); + } if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir +