From 131546002da0dc6d8d511a62599df071279ef347 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 21 Mar 2018 10:51:35 +0800 Subject: [PATCH 001/512] HDFS-13307. RBF: Improve the use of setQuota command. Contributed by liuhongtong. (cherry picked from commit 69fe4407ebd885fed69d9cf7f4dd17d3ba1acb60) --- .../hdfs/tools/federation/RouterAdmin.java | 35 +++++++++++++------ .../federation/router/TestRouterAdminCLI.java | 2 ++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index f4adbad12cd3f..927b073795e3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -453,7 +453,8 @@ private boolean setQuota(String[] parameters, int i) throws IOException { try { nsQuota = Long.parseLong(parameters[i]); } catch (Exception e) { - System.err.println("Cannot parse nsQuota: " + parameters[i]); + throw new IllegalArgumentException( + "Cannot parse nsQuota: " + parameters[i]); } } else if (parameters[i].equals("-ssQuota")) { i++; @@ -461,7 +462,8 @@ private boolean setQuota(String[] parameters, int i) throws IOException { ssQuota = StringUtils.TraditionalBinaryPrefix .string2long(parameters[i]); } catch (Exception e) { - System.err.println("Cannot parse ssQuota: " + parameters[i]); + throw new IllegalArgumentException( + "Cannot parse ssQuota: " + parameters[i]); } } @@ -469,8 +471,14 @@ private boolean setQuota(String[] parameters, int i) throws IOException { } if (nsQuota <= 0 || ssQuota <= 0) { - System.err.println("Input quota value should be a positive number."); - return false; + throw new IllegalArgumentException( + "Input quota value should be a positive number."); + } + + if (nsQuota == HdfsConstants.QUOTA_DONT_SET && + ssQuota == HdfsConstants.QUOTA_DONT_SET) { + throw new IllegalArgumentException( + "Must specify at least one of -nsQuota and -ssQuota."); } return updateQuota(mount, nsQuota, ssQuota); @@ -515,18 +523,23 @@ private boolean updateQuota(String mount, long nsQuota, long ssQuota) } if (existingEntry == null) { - return false; + throw new IOException(mount + " doesn't exist in mount table."); } else { long nsCount = existingEntry.getQuota().getFileAndDirectoryCount(); long ssCount = existingEntry.getQuota().getSpaceConsumed(); - // If nsQuota or ssQuota was unset, reset corresponding usage - // value to zero. - if (nsQuota == HdfsConstants.QUOTA_DONT_SET) { + // If nsQuota and ssQuota were unset, clear nsQuota and ssQuota. + if (nsQuota == HdfsConstants.QUOTA_DONT_SET && + ssQuota == HdfsConstants.QUOTA_DONT_SET) { nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT; - } - - if (nsQuota == HdfsConstants.QUOTA_DONT_SET) { ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT; + } else { + // If nsQuota or ssQuota was unset, use the value in mount table. + if (nsQuota == HdfsConstants.QUOTA_DONT_SET) { + nsQuota = existingEntry.getQuota().getQuota(); + } + if (ssQuota == HdfsConstants.QUOTA_DONT_SET) { + ssQuota = existingEntry.getQuota().getSpaceQuota(); + } } RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder() diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index 6111c6bb3e5a7..d59e25f740f24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -381,6 +381,8 @@ public void testSetAndClearQuota() throws Exception { .getMountTableEntries(getRequest); mountTable = getResponse.getEntries().get(0); quotaUsage = mountTable.getQuota(); + // verify if ns quota keeps quondam value + assertEquals(nsQuota, quotaUsage.getQuota()); // verify if ss quota is correctly set assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota()); From d8764f1c9073871aa801f0a5244c3c0ccdbc29de Mon Sep 17 00:00:00 2001 From: Haibo Chen Date: Tue, 20 Mar 2018 04:36:21 -0700 Subject: [PATCH 002/512] YARN-8053. Add hadoop-distcp in exclusion in hbase-server dependencies for timelineservice-hbase packages. (Rohith Sharma K S via Haibo Chen) (cherry picked from commit 3ff6977d3e29cff455a71f668bec8ed35fb90774) --- .../pom.xml | 8 ++++++++ .../pom.xml | 4 ++++ .../pom.xml | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index 20ed7f0b6114d..5c4a2ac7ef3e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -234,6 +234,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + @@ -267,6 +271,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml index 84e04c5d70862..91efbd5f6e9bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml @@ -149,6 +149,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + org.mortbay.jetty jetty diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml index 430fc753280c8..80a8222d4b13f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml @@ -131,6 +131,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-distcp + org.mortbay.jetty jetty From bbfe36d6865a568d0e709e651a3f48ab9644794b Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 21 Mar 2018 08:11:19 +0530 Subject: [PATCH 003/512] YARN-7581. HBase filters are not constructed correctly in ATSv2. Contributed by Habio Chen. (cherry picked from commit 29acea5000337a7f529bb1810a2af2b0af4d5f1d) --- .../reader/ApplicationEntityReader.java | 14 ++++-- .../reader/FlowActivityEntityReader.java | 4 +- .../storage/reader/FlowRunEntityReader.java | 7 ++- .../storage/reader/GenericEntityReader.java | 12 +++-- .../reader/SubApplicationEntityReader.java | 15 ++++-- .../storage/reader/TimelineEntityReader.java | 46 +++++++++++++++++-- 6 files changed, 83 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java index 7440316c9e282..29ba1845db954 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; @@ -258,7 +259,8 @@ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) { * @throws IOException if any problem occurs while updating filter list. */ private void updateFilterForConfsAndMetricsToRetrieve( - FilterList listBasedOnFields) throws IOException { + FilterList listBasedOnFields, Set cfsInFields) + throws IOException { TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve(); // Please note that if confsToRetrieve is specified, we would have added // CONFS to fields to retrieve in augmentParams() even if not specified. @@ -268,6 +270,8 @@ private void updateFilterForConfsAndMetricsToRetrieve( createFilterForConfsOrMetricsToRetrieve( dataToRetrieve.getConfsToRetrieve(), ApplicationColumnFamily.CONFIGS, ApplicationColumnPrefix.CONFIG)); + cfsInFields.add( + Bytes.toString(ApplicationColumnFamily.CONFIGS.getBytes())); } // Please note that if metricsToRetrieve is specified, we would have added @@ -278,11 +282,14 @@ private void updateFilterForConfsAndMetricsToRetrieve( createFilterForConfsOrMetricsToRetrieve( dataToRetrieve.getMetricsToRetrieve(), ApplicationColumnFamily.METRICS, ApplicationColumnPrefix.METRIC)); + cfsInFields.add( + Bytes.toString(ApplicationColumnFamily.METRICS.getBytes())); } } @Override - protected FilterList constructFilterListBasedOnFields() throws IOException { + protected FilterList constructFilterListBasedOnFields(Set cfsInFields) + throws IOException { if (!needCreateFilterListBasedOnFields()) { // Fetch all the columns. No need of a filter. return null; @@ -303,8 +310,9 @@ protected FilterList constructFilterListBasedOnFields() throws IOException { excludeFieldsFromInfoColFamily(infoColFamilyList); } listBasedOnFields.addFilter(infoColFamilyList); + cfsInFields.add(Bytes.toString(ApplicationColumnFamily.INFO.getBytes())); - updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields); + updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields, cfsInFields); return listBasedOnFields; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java index d0a0f3bb46e84..7b7eef570156d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; @@ -97,7 +98,8 @@ protected FilterList constructFilterListBasedOnFilters() throws IOException { } @Override - protected FilterList constructFilterListBasedOnFields() { + protected FilterList constructFilterListBasedOnFields( + Set cfsInFields) { return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java index 33a2cf67a2772..80d3e9b336361 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.EnumSet; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; @@ -152,7 +154,8 @@ private FilterList updateFixedColumns() { } @Override - protected FilterList constructFilterListBasedOnFields() throws IOException { + protected FilterList constructFilterListBasedOnFields( + Set cfsInFields) throws IOException { FilterList list = new FilterList(Operator.MUST_PASS_ONE); // By default fetch everything in INFO column family. FamilyFilter infoColumnFamily = @@ -166,6 +169,7 @@ protected FilterList constructFilterListBasedOnFields() throws IOException { && !hasField(dataToRetrieve.getFieldsToRetrieve(), Field.METRICS)) { FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE); infoColFamilyList.addFilter(infoColumnFamily); + cfsInFields.add(Bytes.toString(FlowRunColumnFamily.INFO.getBytes())); infoColFamilyList.addFilter(new QualifierFilter(CompareOp.NOT_EQUAL, new BinaryPrefixComparator(FlowRunColumnPrefix.METRIC .getColumnPrefixBytes("")))); @@ -182,6 +186,7 @@ protected FilterList constructFilterListBasedOnFields() throws IOException { && !metricsToRetrieve.getFilterList().isEmpty()) { FilterList infoColFamilyList = new FilterList(); infoColFamilyList.addFilter(infoColumnFamily); + cfsInFields.add(Bytes.toString(FlowRunColumnFamily.INFO.getBytes())); FilterList columnsList = updateFixedColumns(); columnsList.addFilter(TimelineFilterUtils.createHBaseFilterList( FlowRunColumnPrefix.METRIC, metricsToRetrieve)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java index 02eca84f1f26e..6e62f20aa7d9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; @@ -349,7 +350,8 @@ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) { * @throws IOException if any problem occurs while updating filter list. */ private void updateFilterForConfsAndMetricsToRetrieve( - FilterList listBasedOnFields) throws IOException { + FilterList listBasedOnFields, Set cfsInFields) + throws IOException { TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve(); // Please note that if confsToRetrieve is specified, we would have added // CONFS to fields to retrieve in augmentParams() even if not specified. @@ -359,6 +361,7 @@ private void updateFilterForConfsAndMetricsToRetrieve( .createFilterForConfsOrMetricsToRetrieve( dataToRetrieve.getConfsToRetrieve(), EntityColumnFamily.CONFIGS, EntityColumnPrefix.CONFIG)); + cfsInFields.add(Bytes.toString(EntityColumnFamily.CONFIGS.getBytes())); } // Please note that if metricsToRetrieve is specified, we would have added @@ -369,11 +372,13 @@ private void updateFilterForConfsAndMetricsToRetrieve( .createFilterForConfsOrMetricsToRetrieve( dataToRetrieve.getMetricsToRetrieve(), EntityColumnFamily.METRICS, EntityColumnPrefix.METRIC)); + cfsInFields.add(Bytes.toString(EntityColumnFamily.METRICS.getBytes())); } } @Override - protected FilterList constructFilterListBasedOnFields() throws IOException { + protected FilterList constructFilterListBasedOnFields(Set cfsInFields) + throws IOException { if (!needCreateFilterListBasedOnFields()) { // Fetch all the columns. No need of a filter. return null; @@ -394,7 +399,8 @@ protected FilterList constructFilterListBasedOnFields() throws IOException { excludeFieldsFromInfoColFamily(infoColFamilyList); } listBasedOnFields.addFilter(infoColFamilyList); - updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields); + cfsInFields.add(Bytes.toString(EntityColumnFamily.INFO.getBytes())); + updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields, cfsInFields); return listBasedOnFields; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java index faed34857d7a8..6a91c7b46d027 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; @@ -247,7 +248,8 @@ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) { * @throws IOException if any problem occurs while updating filter list. */ private void updateFilterForConfsAndMetricsToRetrieve( - FilterList listBasedOnFields) throws IOException { + FilterList listBasedOnFields, Set cfsInFields) + throws IOException { TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve(); // Please note that if confsToRetrieve is specified, we would have added // CONFS to fields to retrieve in augmentParams() even if not specified. @@ -258,6 +260,8 @@ private void updateFilterForConfsAndMetricsToRetrieve( dataToRetrieve.getConfsToRetrieve(), SubApplicationColumnFamily.CONFIGS, SubApplicationColumnPrefix.CONFIG)); + cfsInFields.add( + Bytes.toString(SubApplicationColumnFamily.CONFIGS.getBytes())); } // Please note that if metricsToRetrieve is specified, we would have added @@ -269,11 +273,14 @@ private void updateFilterForConfsAndMetricsToRetrieve( dataToRetrieve.getMetricsToRetrieve(), SubApplicationColumnFamily.METRICS, SubApplicationColumnPrefix.METRIC)); + cfsInFields.add( + Bytes.toString(SubApplicationColumnFamily.METRICS.getBytes())); } } @Override - protected FilterList constructFilterListBasedOnFields() throws IOException { + protected FilterList constructFilterListBasedOnFields(Set cfsInFields) + throws IOException { if (!needCreateFilterListBasedOnFields()) { // Fetch all the columns. No need of a filter. return null; @@ -293,7 +300,9 @@ protected FilterList constructFilterListBasedOnFields() throws IOException { excludeFieldsFromInfoColFamily(infoColFamilyList); } listBasedOnFields.addFilter(infoColFamilyList); - updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields); + cfsInFields.add( + Bytes.toString(SubApplicationColumnFamily.INFO.getBytes())); + updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields, cfsInFields); return listBasedOnFields; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java index 3168163ed9649..43ba2afe739c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java @@ -30,11 +30,16 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.BinaryPrefixComparator; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FamilyFilter; +import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; @@ -122,11 +127,12 @@ protected TimelineEntityReader(TimelineReaderContext ctxt, * results fetched from HBase back-end storage. This is called only for * multiple entity reads. * + * @param cfsInFields column families in the fields * @return a {@link FilterList} object. * @throws IOException if any problem occurs while creating filter list. */ - protected abstract FilterList constructFilterListBasedOnFields() - throws IOException; + protected abstract FilterList constructFilterListBasedOnFields( + Set cfsInFields) throws IOException; /** * Creates a {@link FilterList} based on info, config and metric filters. This @@ -151,7 +157,9 @@ private FilterList createFilterList() throws IOException { FilterList listBasedOnFilters = constructFilterListBasedOnFilters(); boolean hasListBasedOnFilters = listBasedOnFilters != null && !listBasedOnFilters.getFilters().isEmpty(); - FilterList listBasedOnFields = constructFilterListBasedOnFields(); + Set cfsInListBasedOnFields = new HashSet<>(0); + FilterList listBasedOnFields = + constructFilterListBasedOnFields(cfsInListBasedOnFields); boolean hasListBasedOnFields = listBasedOnFields != null && !listBasedOnFields.getFilters().isEmpty(); // If filter lists based on both filters and fields can be created, @@ -164,6 +172,21 @@ private FilterList createFilterList() throws IOException { if (hasListBasedOnFilters && hasListBasedOnFields) { FilterList list = new FilterList(); list.addFilter(listBasedOnFilters); + + Set cfsInListBasedOnFilters = new HashSet<>(0); + extractColumnFamiliesFromFiltersBasedOnFilters( + listBasedOnFilters, cfsInListBasedOnFilters); + + // must exclude cfs that are already covered in fields-based filters + // otherwise we will return the whole cf + cfsInListBasedOnFilters.removeAll(cfsInListBasedOnFields); + + if (!cfsInListBasedOnFilters.isEmpty()) { + for (String cf: cfsInListBasedOnFilters) { + listBasedOnFields.addFilter(new FamilyFilter(CompareOp.EQUAL, + new BinaryComparator(Bytes.toBytes(cf)))); + } + } list.addFilter(listBasedOnFields); return list; } else if (hasListBasedOnFilters) { @@ -174,6 +197,21 @@ private FilterList createFilterList() throws IOException { return null; } + private static void extractColumnFamiliesFromFiltersBasedOnFilters( + Filter hbaseFilterBasedOnTLSFilter, Set columnFamilies) { + if (hbaseFilterBasedOnTLSFilter instanceof SingleColumnValueFilter) { + byte[] cf = ((SingleColumnValueFilter) + hbaseFilterBasedOnTLSFilter).getFamily(); + columnFamilies.add(Bytes.toString(cf)); + } else if (hbaseFilterBasedOnTLSFilter instanceof FilterList) { + FilterList filterListBase = (FilterList) hbaseFilterBasedOnTLSFilter; + for (Filter fs: filterListBase.getFilters()) { + extractColumnFamiliesFromFiltersBasedOnFilters(fs, columnFamilies); + } + } + } + + protected TimelineDataToRetrieve getDataToRetrieve() { return dataToRetrieve; } @@ -206,7 +244,7 @@ public TimelineEntity readEntity(Configuration hbaseConf, Connection conn) validateParams(); augmentParams(hbaseConf, conn); - FilterList filterList = constructFilterListBasedOnFields(); + FilterList filterList = constructFilterListBasedOnFields(new HashSet<>(0)); if (LOG.isDebugEnabled() && filterList != null) { LOG.debug("FilterList created for get is - " + filterList); } From 0db4b587c604300ffb44677f7cf81ae3faa430d3 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 21 Mar 2018 11:32:05 +0800 Subject: [PATCH 004/512] HDFS-13250. RBF: Router to manage requests across multiple subclusters. Contributed by Inigo Goiri. (cherry picked from commit 2caba999bbb9d6e3ec56024a0a9d3d56a229edcf) --- .../federation/router/RouterRpcClient.java | 60 +++ .../federation/router/RouterRpcServer.java | 136 +++++- .../federation/store/records/MountTable.java | 13 +- .../router/TestRouterAllResolver.java | 402 ++++++++++++++++++ 4 files changed, 598 insertions(+), 13 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index 0d298ac6c4bf2..c973aa68779c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -768,6 +768,66 @@ private static boolean isExpectedValue(Object expectedValue, Object value) { } } + /** + * Invoke method in all locations and return success if any succeeds. + * + * @param locations List of remote locations to call concurrently. + * @param method The remote method and parameters to invoke. + * @return If the call succeeds in any location. + * @throws IOException If any of the calls return an exception. + */ + public boolean invokeAll( + final Collection locations, final RemoteMethod method) + throws IOException { + boolean anyResult = false; + Map results = + invokeConcurrent(locations, method, false, false, Boolean.class); + for (Boolean value : results.values()) { + boolean result = value.booleanValue(); + if (result) { + anyResult = true; + } + } + return anyResult; + } + + /** + * Invoke multiple concurrent proxy calls to different clients. Returns an + * array of results. + * + * Re-throws exceptions generated by the remote RPC call as either + * RemoteException or IOException. + * + * @param The type of the remote location. + * @param locations List of remote locations to call concurrently. + * @param method The remote method and parameters to invoke. + * @throws IOException If all the calls throw an exception. + */ + public void invokeConcurrent( + final Collection locations, final RemoteMethod method) + throws IOException { + invokeConcurrent(locations, method, void.class); + } + + /** + * Invoke multiple concurrent proxy calls to different clients. Returns an + * array of results. + * + * Re-throws exceptions generated by the remote RPC call as either + * RemoteException or IOException. + * + * @param The type of the remote location. + * @param locations List of remote locations to call concurrently. + * @param method The remote method and parameters to invoke. + * @return Result of invoking the method per subcluster: nsId -> result. + * @throws IOException If all the calls throw an exception. + */ + public Map invokeConcurrent( + final Collection locations, final RemoteMethod method, Class clazz) + throws IOException { + return invokeConcurrent(locations, method, false, false, clazz); + } + /** * Invoke multiple concurrent proxy calls to different clients. Returns an * array of results. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index e3a18344f7b05..661022c36ef0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -510,6 +510,18 @@ public HdfsFileStatus create(String src, FsPermission masked, throws IOException { checkOperation(OperationCategory.WRITE); + if (createParent && isPathAll(src)) { + int index = src.lastIndexOf(Path.SEPARATOR); + String parent = src.substring(0, index); + LOG.debug("Creating {} requires creating parent {}", src, parent); + FsPermission parentPermissions = getParentPermission(masked); + boolean success = mkdirs(parent, parentPermissions, createParent); + if (!success) { + // This shouldn't happen as mkdirs returns true or exception + LOG.error("Couldn't create parents for {}", src); + } + } + RemoteLocation createLocation = getCreateLocation(src); RemoteMethod method = new RemoteMethod("create", new Class[] {String.class, FsPermission.class, String.class, @@ -521,6 +533,32 @@ public HdfsFileStatus create(String src, FsPermission masked, return (HdfsFileStatus) rpcClient.invokeSingle(createLocation, method); } + /** + * Get the permissions for the parent of a child with given permissions. If + * the child has r--, we will set it to r-x. + * @param mask The permission mask of the child. + * @return The permission mask of the parent. + */ + private static FsPermission getParentPermission(final FsPermission mask) { + FsPermission ret = new FsPermission( + applyExecute(mask.getUserAction()), + applyExecute(mask.getGroupAction()), + applyExecute(mask.getOtherAction())); + return ret; + } + + /** + * Apply the execute permissions if it can be read. + * @param action Input permission. + * @return Output permission. + */ + private static FsAction applyExecute(final FsAction action) { + if (action.and(FsAction.READ) == FsAction.READ) { + return action.or(FsAction.EXECUTE); + } + return action; + } + /** * Get the location to create a file. It checks if the file already existed * in one of the locations. @@ -580,7 +618,7 @@ public LastBlockWithStatus append(String src, final String clientName, RemoteMethod method = new RemoteMethod("append", new Class[] {String.class, String.class, EnumSetWritable.class}, new RemoteParam(), clientName, flag); - return (LastBlockWithStatus) rpcClient.invokeSequential( + return rpcClient.invokeSequential( locations, method, LastBlockWithStatus.class, null); } @@ -643,7 +681,11 @@ public void setPermission(String src, FsPermission permissions) RemoteMethod method = new RemoteMethod("setPermission", new Class[] {String.class, FsPermission.class}, new RemoteParam(), permissions); - rpcClient.invokeSequential(locations, method); + if (isPathAll(src)) { + rpcClient.invokeConcurrent(locations, method); + } else { + rpcClient.invokeSequential(locations, method); + } } @Override // ClientProtocol @@ -655,7 +697,11 @@ public void setOwner(String src, String username, String groupname) RemoteMethod method = new RemoteMethod("setOwner", new Class[] {String.class, String.class, String.class}, new RemoteParam(), username, groupname); - rpcClient.invokeSequential(locations, method); + if (isPathAll(src)) { + rpcClient.invokeConcurrent(locations, method); + } else { + rpcClient.invokeSequential(locations, method); + } } /** @@ -933,8 +979,12 @@ public boolean delete(String src, boolean recursive) throws IOException { RemoteMethod method = new RemoteMethod("delete", new Class[] {String.class, boolean.class}, new RemoteParam(), recursive); - return ((Boolean) rpcClient.invokeSequential(locations, method, - Boolean.class, Boolean.TRUE)).booleanValue(); + if (isPathAll(src)) { + return rpcClient.invokeAll(locations, method); + } else { + return rpcClient.invokeSequential(locations, method, + Boolean.class, Boolean.TRUE).booleanValue(); + } } @Override // ClientProtocol @@ -943,6 +993,15 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent) checkOperation(OperationCategory.WRITE); final List locations = getLocationsForPath(src, true); + RemoteMethod method = new RemoteMethod("mkdirs", + new Class[] {String.class, FsPermission.class, boolean.class}, + new RemoteParam(), masked, createParent); + + // Create in all locations + if (isPathAll(src)) { + return rpcClient.invokeAll(locations, method); + } + if (locations.size() > 1) { // Check if this directory already exists try { @@ -959,9 +1018,6 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent) } RemoteLocation firstLocation = locations.get(0); - RemoteMethod method = new RemoteMethod("mkdirs", - new Class[] {String.class, FsPermission.class, boolean.class}, - new RemoteParam(), masked, createParent); return ((Boolean) rpcClient.invokeSingle(firstLocation, method)) .booleanValue(); } @@ -1077,8 +1133,16 @@ public HdfsFileStatus getFileInfo(String src) throws IOException { final List locations = getLocationsForPath(src, false); RemoteMethod method = new RemoteMethod("getFileInfo", new Class[] {String.class}, new RemoteParam()); - HdfsFileStatus ret = (HdfsFileStatus) rpcClient.invokeSequential( - locations, method, HdfsFileStatus.class, null); + + HdfsFileStatus ret = null; + // If it's a directory, we check in all locations + if (isPathAll(src)) { + ret = getFileInfoAll(locations, method); + } else { + // Check for file information sequentially + ret = (HdfsFileStatus) rpcClient.invokeSequential( + locations, method, HdfsFileStatus.class, null); + } // If there is no real path, check mount points if (ret == null) { @@ -1096,6 +1160,37 @@ public HdfsFileStatus getFileInfo(String src) throws IOException { return ret; } + /** + * Get the file info from all the locations. + * + * @param locations Locations to check. + * @param method The file information method to run. + * @return The first file info if it's a file, the directory if it's + * everywhere. + * @throws IOException If all the locations throw an exception. + */ + private HdfsFileStatus getFileInfoAll(final List locations, + final RemoteMethod method) throws IOException { + + // Get the file info from everybody + Map results = + rpcClient.invokeConcurrent(locations, method, HdfsFileStatus.class); + + // We return the first file + HdfsFileStatus dirStatus = null; + for (RemoteLocation loc : locations) { + HdfsFileStatus fileStatus = results.get(loc); + if (fileStatus != null) { + if (!fileStatus.isDirectory()) { + return fileStatus; + } else if (dirStatus == null) { + dirStatus = fileStatus; + } + } + } + return dirStatus; + } + @Override // ClientProtocol public boolean isFileClosed(String src) throws IOException { checkOperation(OperationCategory.READ); @@ -2071,6 +2166,27 @@ protected List getLocationsForPath( } } + /** + * Check if a path should be in all subclusters. + * + * @param path Path to check. + * @return If a path should be in all subclusters. + */ + private boolean isPathAll(final String path) { + if (subclusterResolver instanceof MountTableResolver) { + try { + MountTableResolver mountTable = (MountTableResolver)subclusterResolver; + MountTable entry = mountTable.getMountPoint(path); + if (entry != null) { + return entry.isAll(); + } + } catch (IOException e) { + LOG.error("Cannot get mount point: {}", e.getMessage()); + } + } + return false; + } + /** * Check if a path is in a read only mount point. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java index 0eab0439940e0..f8fec87c606fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java @@ -37,8 +37,6 @@ import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Data schema for @@ -50,7 +48,6 @@ */ public abstract class MountTable extends BaseRecord { - private static final Logger LOG = LoggerFactory.getLogger(MountTable.class); public static final String ERROR_MSG_NO_SOURCE_PATH = "Invalid entry, no source path specified "; public static final String ERROR_MSG_MUST_START_WITH_BACK_SLASH = @@ -417,6 +414,16 @@ public boolean equals(Object obj) { return false; } + /** + * Check if a mount table spans all locations. + * @return If the mount table spreads across all locations. + */ + public boolean isAll() { + DestinationOrder order = getDestOrder(); + return order == DestinationOrder.HASH_ALL || + order == DestinationOrder.RANDOM; + } + /** * Normalize a path for that filesystem. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java new file mode 100644 index 0000000000000..90abd019594df --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java @@ -0,0 +1,402 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver; +import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; +import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests the use of the resolvers that write in all subclusters from the + * Router. It supports: + *
  • HashResolver + *
  • RandomResolver. + */ +public class TestRouterAllResolver { + + /** Directory that will be in a HASH_ALL mount point. */ + private static final String TEST_DIR_HASH_ALL = "/hashall"; + /** Directory that will be in a HASH_ALL mount point. */ + private static final String TEST_DIR_RANDOM = "/random"; + + /** Number of namespaces. */ + private static final int NUM_NAMESPACES = 2; + + + /** Mini HDFS clusters with Routers and State Store. */ + private static StateStoreDFSCluster cluster; + /** Router for testing. */ + private static RouterContext routerContext; + /** Router/federated filesystem. */ + private static FileSystem routerFs; + /** Filesystem for each namespace. */ + private static List nsFss = new LinkedList<>(); + + + @Before + public void setup() throws Exception { + // 2 nameservices with 1 namenode each (no HA needed for this test) + cluster = new StateStoreDFSCluster( + false, NUM_NAMESPACES, MultipleDestinationMountTableResolver.class); + + // Start NNs and DNs and wait until ready + cluster.startCluster(); + + // Build and start a Router with: State Store + Admin + RPC + Configuration routerConf = new RouterConfigBuilder() + .stateStore() + .admin() + .rpc() + .build(); + cluster.addRouterOverrides(routerConf); + cluster.startRouters(); + routerContext = cluster.getRandomRouter(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + + // Setup the test mount point + createMountTableEntry(TEST_DIR_HASH_ALL, DestinationOrder.HASH_ALL); + createMountTableEntry(TEST_DIR_RANDOM, DestinationOrder.RANDOM); + + // Get filesystems for federated and each namespace + routerFs = routerContext.getFileSystem(); + for (String nsId : cluster.getNameservices()) { + List nns = cluster.getNamenodes(nsId); + for (NamenodeContext nn : nns) { + FileSystem nnFs = nn.getFileSystem(); + nsFss.add(nnFs); + } + } + assertEquals(NUM_NAMESPACES, nsFss.size()); + } + + @After + public void cleanup() { + cluster.shutdown(); + cluster = null; + routerContext = null; + routerFs = null; + nsFss.clear(); + } + + @Test + public void testHashAll() throws Exception { + testAll(TEST_DIR_HASH_ALL); + } + + @Test + public void testRandomAll() throws Exception { + testAll(TEST_DIR_RANDOM); + } + + /** + * Tests that the resolver spreads files across subclusters in the whole + * tree. + * @throws Exception If the resolver is not working. + */ + private void testAll(final String path) throws Exception { + + // Create directories in different levels + routerFs.mkdirs(new Path(path + "/dir0")); + routerFs.mkdirs(new Path(path + "/dir1")); + routerFs.mkdirs(new Path(path + "/dir2/dir20")); + routerFs.mkdirs(new Path(path + "/dir2/dir21")); + routerFs.mkdirs(new Path(path + "/dir2/dir22")); + routerFs.mkdirs(new Path(path + "/dir2/dir22/dir220")); + routerFs.mkdirs(new Path(path + "/dir2/dir22/dir221")); + routerFs.mkdirs(new Path(path + "/dir2/dir22/dir222")); + assertDirsEverywhere(path, 9); + + // Create 14 files at different levels of the tree + createTestFile(routerFs, path + "/dir0/file1.txt"); + createTestFile(routerFs, path + "/dir0/file2.txt"); + createTestFile(routerFs, path + "/dir1/file2.txt"); + createTestFile(routerFs, path + "/dir1/file3.txt"); + createTestFile(routerFs, path + "/dir2/dir20/file4.txt"); + createTestFile(routerFs, path + "/dir2/dir20/file5.txt"); + createTestFile(routerFs, path + "/dir2/dir21/file6.txt"); + createTestFile(routerFs, path + "/dir2/dir21/file7.txt"); + createTestFile(routerFs, path + "/dir2/dir22/file8.txt"); + createTestFile(routerFs, path + "/dir2/dir22/file9.txt"); + createTestFile(routerFs, path + "/dir2/dir22/dir220/file10.txt"); + createTestFile(routerFs, path + "/dir2/dir22/dir220/file11.txt"); + createTestFile(routerFs, path + "/dir2/dir22/dir220/file12.txt"); + createTestFile(routerFs, path + "/dir2/dir22/dir220/file13.txt"); + assertDirsEverywhere(path, 9); + assertFilesDistributed(path, 14); + + // Test append + String testFile = path + "/dir2/dir22/dir220/file-append.txt"; + createTestFile(routerFs, testFile); + Path testFilePath = new Path(testFile); + assertTrue("Created file is too small", + routerFs.getFileStatus(testFilePath).getLen() > 50); + appendTestFile(routerFs, testFile); + assertTrue("Append file is too small", + routerFs.getFileStatus(testFilePath).getLen() > 110); + assertDirsEverywhere(path, 9); + assertFilesDistributed(path, 15); + + // Removing a directory should remove it from every subcluster + routerFs.delete(new Path(path + "/dir2/dir22/dir220"), true); + assertDirsEverywhere(path, 8); + assertFilesDistributed(path, 10); + + // Removing all sub directories + routerFs.delete(new Path(path + "/dir0"), true); + routerFs.delete(new Path(path + "/dir1"), true); + routerFs.delete(new Path(path + "/dir2"), true); + assertDirsEverywhere(path, 0); + assertFilesDistributed(path, 0); + } + + /** + * Directories in HASH_ALL mount points must be in every namespace. + * @param path Path to check under. + * @param expectedNumDirs Expected number of directories. + * @throws IOException If it cannot check the directories. + */ + private void assertDirsEverywhere(String path, int expectedNumDirs) + throws IOException { + + // Check for the directories in each filesystem + List files = listRecursive(routerFs, path); + int numDirs = 0; + for (FileStatus file : files) { + if (file.isDirectory()) { + numDirs++; + + Path dirPath = file.getPath(); + Path checkPath = getRelativePath(dirPath); + for (FileSystem nsFs : nsFss) { + FileStatus fileStatus1 = nsFs.getFileStatus(checkPath); + assertTrue(file + " should be a directory", + fileStatus1.isDirectory()); + } + } + } + assertEquals(expectedNumDirs, numDirs); + } + + /** + * Check that the files are somewhat spread across namespaces. + * @param path Path to check under. + * @param expectedNumFiles Number of files expected. + * @throws IOException If the files cannot be checked. + */ + private void assertFilesDistributed(String path, int expectedNumFiles) + throws IOException { + + // Check where the files went + List routerFiles = listRecursive(routerFs, path); + List> nssFiles = new LinkedList<>(); + for (FileSystem nsFs : nsFss) { + List nsFiles = listRecursive(nsFs, path); + nssFiles.add(nsFiles); + } + + // We should see all the files in the federated view + int numRouterFiles = getNumTxtFiles(routerFiles); + assertEquals(numRouterFiles, expectedNumFiles); + + // All the files should be spread somewhat evenly across subclusters + List numNsFiles = new LinkedList<>(); + int sumNsFiles = 0; + for (int i = 0; i < NUM_NAMESPACES; i++) { + List nsFiles = nssFiles.get(i); + int numFiles = getNumTxtFiles(nsFiles); + numNsFiles.add(numFiles); + sumNsFiles += numFiles; + } + assertEquals(numRouterFiles, sumNsFiles); + if (expectedNumFiles > 0) { + for (int numFiles : numNsFiles) { + assertTrue("Files not distributed: " + numNsFiles, numFiles > 0); + } + } + } + + /** + * Create a test file in the filesystem and check if it was written. + * @param fs Filesystem. + * @param filename Name of the file to create. + * @throws IOException If it cannot create the file. + */ + private static void createTestFile( + final FileSystem fs, final String filename)throws IOException { + + final Path path = new Path(filename); + + // Write the data + FSDataOutputStream os = fs.create(path); + os.writeUTF("Test data " + filename); + os.close(); + + // Read the data and check + FSDataInputStream is = fs.open(path); + String read = is.readUTF(); + assertEquals("Test data " + filename, read); + is.close(); + } + + /** + * Append to a test file in the filesystem and check if we appended. + * @param fs Filesystem. + * @param filename Name of the file to append to. + * @throws IOException + */ + private static void appendTestFile( + final FileSystem fs, final String filename) throws IOException { + final Path path = new Path(filename); + + // Write the data + FSDataOutputStream os = fs.append(path); + os.writeUTF("Test append data " + filename); + os.close(); + + // Read the data previous data + FSDataInputStream is = fs.open(path); + String read = is.readUTF(); + assertEquals(read, "Test data " + filename); + // Read the new data and check + read = is.readUTF(); + assertEquals(read, "Test append data " + filename); + is.close(); + } + + /** + * Count the number of text files in a list. + * @param files File list. + * @return Number of .txt files. + */ + private static int getNumTxtFiles(final List files) { + int numFiles = 0; + for (FileStatus file : files) { + if (file.getPath().getName().endsWith(".txt")) { + numFiles++; + } + } + return numFiles; + } + + /** + * Get the relative path within a filesystem (removes the filesystem prefix). + * @param path Path to check. + * @return File within the filesystem. + */ + private static Path getRelativePath(final Path path) { + URI uri = path.toUri(); + String uriPath = uri.getPath(); + return new Path(uriPath); + } + + /** + * Get the list the files/dirs under a path. + * @param fs Filesystem to check in. + * @param path Path to check for. + * @return List of files. + * @throws IOException If it cannot list the files. + */ + private List listRecursive( + final FileSystem fs, final String path) throws IOException { + List ret = new LinkedList<>(); + List temp = new LinkedList<>(); + temp.add(new Path(path)); + while (!temp.isEmpty()) { + Path p = temp.remove(0); + for (FileStatus fileStatus : fs.listStatus(p)) { + ret.add(fileStatus); + if (fileStatus.isDirectory()) { + temp.add(fileStatus.getPath()); + } + } + } + return ret; + } + + /** + * Add a mount table entry in all nameservices and wait until it is + * available in all routers. + * @param mountPoint Name of the mount point. + * @param order Order of the mount table entry. + * @throws Exception If the entry could not be created. + */ + private void createMountTableEntry( + final String mountPoint, final DestinationOrder order) throws Exception { + + RouterClient admin = routerContext.getAdminClient(); + MountTableManager mountTable = admin.getMountTableManager(); + Map destMap = new HashMap<>(); + for (String nsId : cluster.getNameservices()) { + destMap.put(nsId, mountPoint); + } + MountTable newEntry = MountTable.newInstance(mountPoint, destMap); + newEntry.setDestOrder(order); + AddMountTableEntryRequest addRequest = + AddMountTableEntryRequest.newInstance(newEntry); + AddMountTableEntryResponse addResponse = + mountTable.addMountTableEntry(addRequest); + boolean created = addResponse.getStatus(); + assertTrue(created); + + // Refresh the caches to get the mount table + Router router = routerContext.getRouter(); + StateStoreService stateStore = router.getStateStore(); + stateStore.refreshCaches(true); + + // Check for the path + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(mountPoint); + GetMountTableEntriesResponse getResponse = + mountTable.getMountTableEntries(getRequest); + List entries = getResponse.getEntries(); + assertEquals(1, entries.size()); + assertEquals(mountPoint, entries.get(0).getSourcePath()); + } +} \ No newline at end of file From 5d4b2c31a19b2ce7d94a9ca12ed6a1def428bb83 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 21 Mar 2018 17:19:20 -0500 Subject: [PATCH 005/512] YARN-8054. Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread. Contributed by Jonathan Eagles (cherry picked from commit 5aa7052e319c3273243dda9993fb6c2d776eb7cc) --- .../yarn/server/nodemanager/LocalDirsHandlerService.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index c0630b246f630..621cabc1b80ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -166,7 +166,12 @@ public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException { @Override public void run() { - checkDirs(); + try { + checkDirs(); + } catch (Throwable t) { + // Prevent uncaught exceptions from killing this thread + LOG.warn("Error while checking local directories: ", t); + } } } From 21db4e9efe9ff634c905aa32667d9dbb203b3699 Mon Sep 17 00:00:00 2001 From: Konstantin V Shvachko Date: Wed, 21 Mar 2018 16:46:03 -0700 Subject: [PATCH 006/512] HDFS-12884. BlockUnderConstructionFeature.truncateBlock should be of type BlockInfo. Contributed by chencan. (cherry picked from commit 8d898ab25f1c2032a07c9bbd96ba3d0c4eb5be87) --- .../blockmanagement/BlockUnderConstructionFeature.java | 6 +++--- .../apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java | 5 +++-- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +- .../java/org/apache/hadoop/hdfs/server/namenode/INode.java | 6 ++---- .../hadoop/hdfs/server/namenode/snapshot/FileDiffList.java | 3 +-- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java index 61390d9fe835c..91023c68972c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java @@ -60,7 +60,7 @@ public class BlockUnderConstructionFeature { /** * The block source to use in the event of copy-on-write truncate. */ - private Block truncateBlock; + private BlockInfo truncateBlock; public BlockUnderConstructionFeature(Block blk, BlockUCState state, DatanodeStorageInfo[] targets, BlockType blockType) { @@ -193,11 +193,11 @@ public long getBlockRecoveryId() { } /** Get recover block */ - public Block getTruncateBlock() { + public BlockInfo getTruncateBlock() { return truncateBlock; } - public void setTruncateBlock(Block recoveryBlock) { + public void setTruncateBlock(BlockInfo recoveryBlock) { this.truncateBlock = recoveryBlock; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java index 034812058eea9..bf55d30591074 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java @@ -104,7 +104,7 @@ static TruncateResult truncate(final FSNamesystem fsn, final String srcArg, final BlockInfo last = file.getLastBlock(); if (last != null && last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) { - final Block truncatedBlock = last.getUnderConstructionFeature() + final BlockInfo truncatedBlock = last.getUnderConstructionFeature() .getTruncateBlock(); if (truncatedBlock != null) { final long truncateLength = file.computeFileSize(false, false) @@ -259,7 +259,8 @@ static Block prepareFileForTruncate(FSNamesystem fsn, INodesInPath iip, oldBlock = file.getLastBlock(); assert !oldBlock.isComplete() : "oldBlock should be under construction"; BlockUnderConstructionFeature uc = oldBlock.getUnderConstructionFeature(); - uc.setTruncateBlock(new Block(oldBlock)); + uc.setTruncateBlock(new BlockInfoContiguous(oldBlock, + oldBlock.getReplication())); uc.getTruncateBlock().setNumBytes(oldBlock.getNumBytes() - lastBlockDelta); uc.getTruncateBlock().setGenerationStamp(newBlock.getGenerationStamp()); truncatedBlockUC = oldBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 868f66967e5ee..752c830f95898 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3376,7 +3376,7 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, BlockUnderConstructionFeature uc = lastBlock.getUnderConstructionFeature(); // determine if last block was intended to be truncated - Block recoveryBlock = uc.getTruncateBlock(); + BlockInfo recoveryBlock = uc.getTruncateBlock(); boolean truncateRecovery = recoveryBlock != null; boolean copyOnTruncate = truncateRecovery && recoveryBlock.getBlockId() != lastBlock.getBlockId(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index e9490fd58fddf..207d97726b1ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; @@ -1054,12 +1053,11 @@ public void addDeleteBlock(BlockInfo toDelete) { if(uc == null) { return; } - Block truncateBlock = uc.getTruncateBlock(); + BlockInfo truncateBlock = uc.getTruncateBlock(); if(truncateBlock == null || truncateBlock.equals(toDelete)) { return; } - assert truncateBlock instanceof BlockInfo : "should be BlockInfo"; - addDeleteBlock((BlockInfo) truncateBlock); + addDeleteBlock(truncateBlock); } public void addUpdateReplicationFactor(BlockInfo block, short targetRepl) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java index 2c04a49ca4c8e..1677534b3c0f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; @@ -129,7 +128,7 @@ void combineAndCollectSnapshotBlocks( } // Check if last block is part of truncate recovery BlockInfo lastBlock = file.getLastBlock(); - Block dontRemoveBlock = null; + BlockInfo dontRemoveBlock = null; if (lastBlock != null && lastBlock.getBlockUCState().equals( HdfsServerConstants.BlockUCState.UNDER_RECOVERY)) { dontRemoveBlock = lastBlock.getUnderConstructionFeature() From 485348a337cd1923ab51784784f560f82c11af26 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Thu, 22 Mar 2018 09:21:52 -0700 Subject: [PATCH 007/512] HDFS-13318. RBF: Fix FindBugs in hadoop-hdfs-rbf. Contributed by Ekanth S. (cherry picked from commit 3d0ee8b80438bfbdf7e5af853b9044a63bd1352c) --- .../federation/resolver/NamenodePriorityComparator.java | 5 ++++- .../hadoop/hdfs/server/federation/router/RemoteMethod.java | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java index fe82f29fbaa62..e9724a3dee776 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodePriorityComparator.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.resolver; +import java.io.Serializable; import java.util.Comparator; /** @@ -31,7 +32,9 @@ * breaker, newest has priority. Expired NNs are excluded. */ public class NamenodePriorityComparator - implements Comparator { + implements Comparator, Serializable { + + private static final long serialVersionUID = 2304924292036293331L; @Override public int compare(FederationNamenodeContext o1, diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java index cd57d45779874..7978105584ede 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java @@ -76,7 +76,7 @@ public RemoteMethod(String method, Class[] pTypes, Object... pParams) } this.params = pParams; - this.types = pTypes; + this.types = Arrays.copyOf(pTypes, pTypes.length); this.methodName = method; } @@ -111,7 +111,7 @@ public Method getMethod() throws IOException { * @return An array of calling types. */ public Class[] getTypes() { - return this.types; + return Arrays.copyOf(this.types, this.types.length); } /** From c245eceddad8883103df30bbff9a285798f29fdf Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 22 Mar 2018 12:28:34 -0500 Subject: [PATCH 008/512] HDFS-13195. DataNode conf page cannot display the current value after reconfig. Contributed by maobaolong. (cherry picked from commit 66461ed98b0b2f630b682bff927dbd74c251c26e) --- .../hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index 2e46b284ac6e9..0ce327a69e192 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -120,6 +120,7 @@ public DatanodeHttpServer(final Configuration conf, this.infoServer = builder.build(); + this.infoServer.setAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE, conf); this.infoServer.setAttribute("datanode", datanode); this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf); this.infoServer.addServlet(null, "/blockScannerReport", From d82e846fa43a441af86355d95a5464c7ebcd5c15 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 22 Mar 2018 10:15:30 -0700 Subject: [PATCH 009/512] HADOOP-15334. Upgrade Maven surefire plugin. Contributed by Arpit Agarwal. --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index a352e5e3db23c..1f2c309dab247 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -115,7 +115,7 @@ -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError - 2.20.1 + 2.21.0 ${maven-surefire-plugin.version} ${maven-surefire-plugin.version} From 59ab86311cd02ef37d08291a4ca63db92bf24521 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Thu, 22 Mar 2018 13:32:57 -0700 Subject: [PATCH 010/512] HDFS-12792. RBF: Test Router-based federation using HDFSContract. Contributed by Inigo Goiri. (cherry picked from commit e196d158a275bfdcfaf372e0f5bcbbc8bbe24106) --- .../contract/router/RouterHDFSContract.java | 98 +++++++++++++++ .../router/TestRouterHDFSContractAppend.java | 44 +++++++ .../router/TestRouterHDFSContractConcat.java | 51 ++++++++ .../router/TestRouterHDFSContractCreate.java | 48 ++++++++ .../router/TestRouterHDFSContractDelete.java | 48 ++++++++ .../TestRouterHDFSContractGetFileStatus.java | 49 ++++++++ .../router/TestRouterHDFSContractMkdir.java | 48 ++++++++ .../router/TestRouterHDFSContractOpen.java | 48 ++++++++ .../router/TestRouterHDFSContractRename.java | 48 ++++++++ .../TestRouterHDFSContractRootDirectory.java | 64 ++++++++++ .../router/TestRouterHDFSContractSeek.java | 49 ++++++++ .../TestRouterHDFSContractSetTimes.java | 49 ++++++++ .../federation/FederationTestUtils.java | 31 +++++ ...Cluster.java => MiniRouterDFSCluster.java} | 37 ++++-- .../federation/StateStoreDFSCluster.java | 2 +- .../federation/router/TestRouterAdmin.java | 2 +- .../federation/router/TestRouterAdminCLI.java | 2 +- .../router/TestRouterAllResolver.java | 4 +- .../router/TestRouterMountTable.java | 4 +- ....java => TestRouterNamenodeHeartbeat.java} | 12 +- .../router/TestRouterNamenodeMonitoring.java | 4 +- .../federation/router/TestRouterQuota.java | 4 +- .../router/TestRouterRPCClientRetries.java | 4 +- .../federation/router/TestRouterRpc.java | 14 +-- .../router/TestRouterRpcMultiDestination.java | 8 +- .../src/test/resources/contract/hdfs.xml | 114 ++++++++++++++++++ 26 files changed, 849 insertions(+), 37 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppend.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcat.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreate.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDelete.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatus.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdir.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpen.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRename.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectory.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeek.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimes.java rename hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/{RouterDFSCluster.java => MiniRouterDFSCluster.java} (96%) rename hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/{TestNamenodeHeartbeat.java => TestRouterNamenodeHeartbeat.java} (93%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/hdfs.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java new file mode 100644 index 0000000000000..97a426e2c2a8c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/RouterHDFSContract.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract.router; + +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; +import org.apache.hadoop.fs.contract.hdfs.HDFSContract; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.junit.Assert; + +/** + * The contract of Router-based Federated HDFS. + */ +public class RouterHDFSContract extends HDFSContract { + + public static final int BLOCK_SIZE = + AbstractFSContractTestBase.TEST_FILE_LEN; + private static MiniRouterDFSCluster cluster; + + public RouterHDFSContract(Configuration conf) { + super(conf); + } + + public static void createCluster() throws IOException { + try { + cluster = new MiniRouterDFSCluster(true, 2); + + // Start NNs and DNs and wait until ready + cluster.startCluster(); + + // Start routers with only an RPC service + cluster.startRouters(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + + // Setup the mount table + cluster.installMockLocations(); + + // Making one Namenodes active per nameservice + if (cluster.isHighAvailability()) { + for (String ns : cluster.getNameservices()) { + cluster.switchToActive(ns, NAMENODES[0]); + cluster.switchToStandby(ns, NAMENODES[1]); + } + } + + cluster.waitActiveNamespaces(); + } catch (Exception e) { + destroyCluster(); + throw new IOException("Cannot start federated cluster", e); + } + } + + public static void destroyCluster() throws IOException { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + public static MiniDFSCluster getCluster() { + return cluster.getCluster(); + } + + public static FileSystem getFileSystem() throws IOException { + //assumes cluster is not null + Assert.assertNotNull("cluster not created", cluster); + return cluster.getRandomRouter().getFileSystem(); + } + + @Override + public FileSystem getTestFileSystem() throws IOException { + return getFileSystem(); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppend.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppend.java new file mode 100644 index 0000000000000..5a9395757fc07 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractAppend.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ + +package org.apache.hadoop.fs.contract.router; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * Test append operations on the Router-based FS. + */ +public class TestRouterHDFSContractAppend extends AbstractContractAppendTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcat.java new file mode 100644 index 0000000000000..96ee71894525c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractConcat.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractContractConcatTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test concat operations on the Router-based FS. + */ +public class TestRouterHDFSContractConcat extends AbstractContractConcatTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + // perform a simple operation on the cluster to verify it is up + RouterHDFSContract.getFileSystem().getDefaultBlockSize(new Path("/")); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreate.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreate.java new file mode 100644 index 0000000000000..530b3068ef0f9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractCreate.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test create operations on the Router-based FS. + */ +public class TestRouterHDFSContractCreate extends AbstractContractCreateTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDelete.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDelete.java new file mode 100644 index 0000000000000..a7d488127ad19 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractDelete.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test delete operations on the Router-based FS. + */ +public class TestRouterHDFSContractDelete extends AbstractContractDeleteTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatus.java new file mode 100644 index 0000000000000..b06c570f70429 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractGetFileStatus.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test get file status operations on the Router-based FS. + */ +public class TestRouterHDFSContractGetFileStatus + extends AbstractContractGetFileStatusTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdir.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdir.java new file mode 100644 index 0000000000000..8c683163ccab6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractMkdir.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on the Router-based FS. + */ +public class TestRouterHDFSContractMkdir extends AbstractContractMkdirTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpen.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpen.java new file mode 100644 index 0000000000000..5e8826e96e4df --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractOpen.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test open operations on the Router-based FS. + */ +public class TestRouterHDFSContractOpen extends AbstractContractOpenTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRename.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRename.java new file mode 100644 index 0000000000000..a90fe0a1e4c7f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRename.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test rename operations on the Router-based FS. + */ +public class TestRouterHDFSContractRename extends AbstractContractRenameTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectory.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectory.java new file mode 100644 index 0000000000000..cc603ddd7bb8c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractRootDirectory.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test root dir operations on the Router-based FS. + */ +public class TestRouterHDFSContractRootDirectory extends + AbstractContractRootDirectoryTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } + + @Override + public void testListEmptyRootDirectory() throws IOException { + // It doesn't apply because we still have the mount points here + } + + @Override + public void testRmEmptyRootDirNonRecursive() throws IOException { + // It doesn't apply because we still have the mount points here + } + + @Override + public void testRecursiveRootListing() throws IOException { + // It doesn't apply because we still have the mount points here + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeek.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeek.java new file mode 100644 index 0000000000000..587704241c6af --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSeek.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test seek operations on the Router-based FS. + */ +public class TestRouterHDFSContractSeek extends AbstractContractSeekTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimes.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimes.java new file mode 100644 index 0000000000000..e7d157e81b174 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/TestRouterHDFSContractSetTimes.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSetTimesTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test set times operations on the Router-based FS. + */ +public class TestRouterHDFSContractSetTimes + extends AbstractContractSetTimesTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterHDFSContract(conf); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java index 0ead93ee7fe4a..b138e4d080b66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java @@ -147,6 +147,37 @@ public Boolean get() { }, 1000, 20 * 1000); } + /** + * Wait for a namenode to be registered with a particular state. + * @param resolver Active namenode resolver. + * @param nsId Nameservice identifier. + * @param state State to check for. + * @throws Exception Failed to verify State Store registration of namenode + * nsId for state. + */ + public static void waitNamenodeRegistered( + final ActiveNamenodeResolver resolver, final String nsId, + final FederationNamenodeServiceState state) throws Exception { + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + List nns = + resolver.getNamenodesForNameserviceId(nsId); + for (FederationNamenodeContext nn : nns) { + if (nn.getState().equals(state)) { + return true; + } + } + } catch (IOException e) { + // Ignore + } + return false; + } + }, 1000, 20 * 1000); + } + public static boolean verifyDate(Date d1, Date d2, long precision) { return Math.abs(d1.getTime() - d2.getTime()) < precision; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java similarity index 96% rename from hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java rename to hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index 9788683b4bac4..df9f038156c7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -42,6 +42,7 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS; import static org.junit.Assert.assertEquals; @@ -75,6 +76,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport; import org.apache.hadoop.hdfs.server.federation.router.Router; @@ -91,15 +93,16 @@ /** * Test utility to mimic a federated HDFS cluster with multiple routers. */ -public class RouterDFSCluster { +public class MiniRouterDFSCluster { private static final Logger LOG = - LoggerFactory.getLogger(RouterDFSCluster.class); + LoggerFactory.getLogger(MiniRouterDFSCluster.class); public static final String TEST_STRING = "teststring"; public static final String TEST_DIR = "testdir"; public static final String TEST_FILE = "testfile"; + private static final Random RND = new Random(); /** Nameservices in the federated cluster. */ private List nameservices; @@ -345,7 +348,8 @@ public String getConfSuffix() { } } - public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes, + public MiniRouterDFSCluster( + boolean ha, int numNameservices, int numNamenodes, long heartbeatInterval, long cacheFlushInterval) { this.highAvailability = ha; this.heartbeatInterval = heartbeatInterval; @@ -353,12 +357,13 @@ public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes, configureNameservices(numNameservices, numNamenodes); } - public RouterDFSCluster(boolean ha, int numNameservices) { + public MiniRouterDFSCluster(boolean ha, int numNameservices) { this(ha, numNameservices, 2, DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS); } - public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) { + public MiniRouterDFSCluster( + boolean ha, int numNameservices, int numNamenodes) { this(ha, numNameservices, numNamenodes, DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS); } @@ -480,6 +485,9 @@ public Configuration generateRouterConfiguration(String nsId, String nnId) { conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS, MockResolver.class, FileSubclusterResolver.class); + // Disable safemode on startup + conf.setBoolean(DFS_ROUTER_SAFEMODE_ENABLE, false); + // Set the nameservice ID for the default NN monitor conf.set(DFS_NAMESERVICE_ID, nsId); if (nnId != null) { @@ -549,8 +557,7 @@ public String getNameservicesKey() { } public String getRandomNameservice() { - Random r = new Random(); - int randIndex = r.nextInt(nameservices.size()); + int randIndex = RND.nextInt(nameservices.size()); return nameservices.get(randIndex); } @@ -772,6 +779,22 @@ public void waitRouterRegistrationQuorum(RouterContext router, waitNamenodeRegistered(nnResolver, nsId, nnId, state); } + /** + * Wait for name spaces to be active. + * @throws Exception If we cannot check the status or we timeout. + */ + public void waitActiveNamespaces() throws Exception { + for (RouterContext r : this.routers) { + Router router = r.router; + final ActiveNamenodeResolver resolver = router.getNamenodeResolver(); + for (FederationNamespaceInfo ns : resolver.getNamespaces()) { + final String nsId = ns.getNameserviceId(); + waitNamenodeRegistered( + resolver, nsId, FederationNamenodeServiceState.ACTIVE); + } + } + } + /** * Get the federated path for a nameservice. * @param nsId Nameservice identifier. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java index aa1906fdd3125..bf63b18469589 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/StateStoreDFSCluster.java @@ -42,7 +42,7 @@ * Test utility to mimic a federated HDFS cluster with a router and a state * store. */ -public class StateStoreDFSCluster extends RouterDFSCluster { +public class StateStoreDFSCluster extends MiniRouterDFSCluster { private static final Class DEFAULT_FILE_RESOLVER = MountTableResolver.class; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index a8ffded3dc73b..3c1a136ce11e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -29,7 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index d59e25f740f24..b36e4342aae34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -30,7 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java index 90abd019594df..4995de4115ee1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAllResolver.java @@ -34,8 +34,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java index 8702b3c87264a..c9e28b1e10f3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java @@ -28,8 +28,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java similarity index 93% rename from hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java rename to hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java index 877fb02f44388..d2bc5d6eb2fcb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestNamenodeHeartbeat.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java @@ -27,8 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.MockResolver; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; import org.apache.hadoop.service.Service.STATE; @@ -42,9 +42,9 @@ * Test the service that heartbeats the state of the namenodes to the State * Store. */ -public class TestNamenodeHeartbeat { +public class TestRouterNamenodeHeartbeat { - private static RouterDFSCluster cluster; + private static MiniRouterDFSCluster cluster; private static ActiveNamenodeResolver namenodeResolver; private static List services; @@ -54,7 +54,7 @@ public class TestNamenodeHeartbeat { @BeforeClass public static void globalSetUp() throws Exception { - cluster = new RouterDFSCluster(true, 2); + cluster = new MiniRouterDFSCluster(true, 2); // Start NNs and DNs and wait until ready cluster.startCluster(); @@ -91,7 +91,7 @@ public static void tearDown() throws IOException { @Test public void testNamenodeHeartbeatService() throws IOException { - RouterDFSCluster testCluster = new RouterDFSCluster(true, 1); + MiniRouterDFSCluster testCluster = new MiniRouterDFSCluster(true, 1); Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration( NAMESERVICES[0]); NamenodeHeartbeatService server = new NamenodeHeartbeatService( diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java index eabc0fe0f521d..75c267643f9a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java @@ -28,8 +28,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index 66a955ead1da8..569e51da91157 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -38,8 +38,8 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java index 61e7657a75da0..1e0f9a17b5008 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java @@ -31,8 +31,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index 2b12a254c455f..e8341a2a06eae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -23,7 +23,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists; -import static org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.TEST_STRING; +import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -72,9 +72,9 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.io.EnumSetWritable; @@ -114,7 +114,7 @@ public int compare( }; /** Federated HDFS cluster. */ - private static RouterDFSCluster cluster; + private static MiniRouterDFSCluster cluster; /** Random Router for this federated cluster. */ private RouterContext router; @@ -142,7 +142,7 @@ public int compare( @BeforeClass public static void globalSetUp() throws Exception { - cluster = new RouterDFSCluster(false, 2); + cluster = new MiniRouterDFSCluster(false, 2); // We need 6 DNs to test Erasure Coding with RS-6-3-64k cluster.setNumDatanodesPerNameservice(6); @@ -221,7 +221,7 @@ public void testRpcService() throws IOException { testRouter.close(); } - protected RouterDFSCluster getCluster() { + protected MiniRouterDFSCluster getCluster() { return TestRouterRpc.cluster; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index 548969162219b..7e0976016be23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -38,9 +38,9 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.federation.MockResolver; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext; -import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; @@ -54,7 +54,7 @@ public class TestRouterRpcMultiDestination extends TestRouterRpc { @Override public void testSetup() throws Exception { - RouterDFSCluster cluster = getCluster(); + MiniRouterDFSCluster cluster = getCluster(); // Create mock locations getCluster().installMockLocations(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/hdfs.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/hdfs.xml new file mode 100644 index 0000000000000..261d4ba136508 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/hdfs.xml @@ -0,0 +1,114 @@ + + + + + + + fs.contract.test.root-tests-enabled + true + + + + fs.file.contract.test.random-seek-count + 500 + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.supports-append + true + + + + fs.contract.supports-atomic-directory-delete + true + + + + fs.contract.supports-atomic-rename + true + + + + fs.contract.supports-block-locality + true + + + + fs.contract.supports-concat + true + + + + fs.contract.supports-seek + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + true + + + + fs.contract.rename-returns-false-if-dest-exists + true + + + + fs.contract.rename-returns-false-if-source-missing + true + + + + fs.contract.supports-settimes + true + + + + fs.contract.supports-getfilestatus + true + + + + fs.contract.supports-file-reference + true + + + + fs.contract.supports-content-check + true + + + From e10e6c90129f5d2f1ecdbaf18a43c529574505f4 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 23 Mar 2018 10:54:00 +0530 Subject: [PATCH 011/512] YARN-8063. DistributedShellTimelinePlugin wrongly check for entityId instead of entityType. Contributed by Rohith Sharma K S. (cherry picked from commit 22c5ddb7c4fb48d5bf5a7456d0b1b27d48c2a485) --- .../distributedshell/DistributedShellTimelinePlugin.java | 2 +- .../distributedshell/TestDistributedShell.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java index 119fa6f3bd503..cf5bc839da1a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java @@ -52,7 +52,7 @@ public Set getTimelineEntityGroupId(String entityType, @Override public Set getTimelineEntityGroupId(String entityId, String entityType) { - if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityId)) { + if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityType)) { ContainerId containerId = ContainerId.fromString(entityId); ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 667b60d21ff07..0e94ad5a2f8dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -494,6 +494,15 @@ private void checkTimelineV1(boolean haveDomain) throws Exception { Assert.assertEquals(2, entities.getEntities().size()); Assert.assertEquals(entities.getEntities().get(0).getEntityType() .toString(), ApplicationMaster.DSEntity.DS_CONTAINER.toString()); + + String entityId = entities.getEntities().get(0).getEntityId(); + org.apache.hadoop.yarn.api.records.timeline.TimelineEntity entity = + yarnCluster.getApplicationHistoryServer().getTimelineStore() + .getEntity(entityId, + ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null); + Assert.assertNotNull(entity); + Assert.assertEquals(entityId, entity.getEntityId()); + if (haveDomain) { Assert.assertEquals(domain.getId(), entities.getEntities().get(0).getDomainId()); From 91791a8abdf3d46278ee7c15eeb3dddc43918742 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Fri, 23 Mar 2018 12:16:30 +0530 Subject: [PATCH 012/512] YARN-7986. ATSv2 REST API queries do not return results for uppercase application tags. Contributed by Charan Hebri. (cherry picked from commit 75fc05f369929db768b767d79351bca8c13ad9ba) --- .../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md index 28c4a9105120b..f097b60b336f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md @@ -486,6 +486,8 @@ You can provide the flow context via YARN application tags: appContext.setApplicationTags(tags); +Note : The Resource Manager converts YARN application tags to lowercase before storing them. Hence one should convert +Flow names and Flow versions to lowercase before using them in REST API queries. ## Timeline Service v.2 REST API From d17aa8d0528a499e594aec4cb3e74651283f3479 Mon Sep 17 00:00:00 2001 From: weiy Date: Fri, 23 Mar 2018 08:32:25 -0700 Subject: [PATCH 013/512] HDFS-12512. RBF: Add WebHDFS. (cherry picked from commit 6e31a090842f8aeedb331b653b075499f8df6c60) --- .../federation/router/RouterHttpServer.java | 4 + .../federation/router/RouterRpcServer.java | 2 +- .../router/RouterWebHdfsMethods.java | 655 ++++++++++++++++++ .../router/web/RouterWebHDFSContract.java | 129 ++++ .../web/TestRouterWebHDFSContractAppend.java | 45 ++ .../web/TestRouterWebHDFSContractConcat.java | 52 ++ .../web/TestRouterWebHDFSContractCreate.java | 49 ++ .../web/TestRouterWebHDFSContractDelete.java | 49 ++ .../web/TestRouterWebHDFSContractMkdir.java | 48 ++ .../web/TestRouterWebHDFSContractOpen.java | 63 ++ .../web/TestRouterWebHDFSContractRename.java | 49 ++ ...estRouterWebHDFSContractRootDirectory.java | 64 ++ .../web/TestRouterWebHDFSContractSeek.java | 63 ++ .../fs/contract/router/web/package-info.java | 22 + .../federation/MiniRouterDFSCluster.java | 5 + .../src/test/resources/contract/webhdfs.xml | 26 + .../server/namenode/NameNodeHttpServer.java | 27 +- .../web/resources/NamenodeWebHdfsMethods.java | 41 +- 18 files changed, 1368 insertions(+), 25 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractAppend.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractConcat.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractCreate.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractDelete.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractMkdir.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractOpen.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRename.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractSeek.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/package-info.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/webhdfs.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java index 93a9cff20e155..21a9871727137 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.service.AbstractService; @@ -86,6 +87,9 @@ protected void serviceStart() throws Exception { this.httpServer = builder.build(); + NameNodeHttpServer.initWebHdfs(conf, httpAddress.getHostName(), httpServer, + RouterWebHdfsMethods.class.getPackage().getName()); + this.httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, this.router); this.httpServer.setAttribute(JspHelper.CURRENT_CONF, this.conf); setupServlets(this.httpServer, this.conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 661022c36ef0e..b3f677d80a572 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -567,7 +567,7 @@ private static FsAction applyExecute(final FsAction action) { * @return The remote location for this file. * @throws IOException If the file has no creation location. */ - private RemoteLocation getCreateLocation(final String src) + protected RemoteLocation getCreateLocation(final String src) throws IOException { final List locations = getLocationsForPath(src, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java new file mode 100644 index 0000000000000..5e9d4d05068a2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -0,0 +1,655 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; +import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; +import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import com.sun.jersey.spi.container.ResourceFilters; +import org.apache.hadoop.hdfs.web.JsonUtil; +import org.apache.hadoop.hdfs.web.ParamFilter; +import org.apache.hadoop.hdfs.web.URLConnectionFactory; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.apache.hadoop.hdfs.web.resources.AccessTimeParam; +import org.apache.hadoop.hdfs.web.resources.AclPermissionParam; +import org.apache.hadoop.hdfs.web.resources.BlockSizeParam; +import org.apache.hadoop.hdfs.web.resources.BufferSizeParam; +import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam; +import org.apache.hadoop.hdfs.web.resources.CreateFlagParam; +import org.apache.hadoop.hdfs.web.resources.CreateParentParam; +import org.apache.hadoop.hdfs.web.resources.DelegationParam; +import org.apache.hadoop.hdfs.web.resources.DestinationParam; +import org.apache.hadoop.hdfs.web.resources.DoAsParam; +import org.apache.hadoop.hdfs.web.resources.ExcludeDatanodesParam; +import org.apache.hadoop.hdfs.web.resources.FsActionParam; +import org.apache.hadoop.hdfs.web.resources.GetOpParam; +import org.apache.hadoop.hdfs.web.resources.GroupParam; +import org.apache.hadoop.hdfs.web.resources.HttpOpParam; +import org.apache.hadoop.hdfs.web.resources.LengthParam; +import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam; +import org.apache.hadoop.hdfs.web.resources.NewLengthParam; +import org.apache.hadoop.hdfs.web.resources.NoRedirectParam; +import org.apache.hadoop.hdfs.web.resources.OffsetParam; +import org.apache.hadoop.hdfs.web.resources.OldSnapshotNameParam; +import org.apache.hadoop.hdfs.web.resources.OverwriteParam; +import org.apache.hadoop.hdfs.web.resources.OwnerParam; +import org.apache.hadoop.hdfs.web.resources.Param; +import org.apache.hadoop.hdfs.web.resources.PermissionParam; +import org.apache.hadoop.hdfs.web.resources.PostOpParam; +import org.apache.hadoop.hdfs.web.resources.PutOpParam; +import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam; +import org.apache.hadoop.hdfs.web.resources.RenewerParam; +import org.apache.hadoop.hdfs.web.resources.ReplicationParam; +import org.apache.hadoop.hdfs.web.resources.SnapshotNameParam; +import org.apache.hadoop.hdfs.web.resources.StartAfterParam; +import org.apache.hadoop.hdfs.web.resources.StoragePolicyParam; +import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam; +import org.apache.hadoop.hdfs.web.resources.TokenKindParam; +import org.apache.hadoop.hdfs.web.resources.TokenServiceParam; +import org.apache.hadoop.hdfs.web.resources.UnmaskedPermissionParam; +import org.apache.hadoop.hdfs.web.resources.UriFsPathParam; +import org.apache.hadoop.hdfs.web.resources.UserParam; +import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam; +import org.apache.hadoop.hdfs.web.resources.XAttrNameParam; +import org.apache.hadoop.hdfs.web.resources.XAttrSetFlagParam; +import org.apache.hadoop.hdfs.web.resources.XAttrValueParam; +import org.apache.hadoop.ipc.ExternalCall; +import org.apache.hadoop.ipc.RetriableException; +import org.apache.hadoop.net.Node; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLDecoder; +import java.security.PrivilegedAction; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Random; + +/** + * WebHDFS Router implementation. This is an extension of + * {@link NamenodeWebHdfsMethods}, and tries to reuse as much as possible. + */ +@Path("") +@ResourceFilters(ParamFilter.class) +public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods { + private static final Logger LOG = + LoggerFactory.getLogger(RouterWebHdfsMethods.class); + + private static final ThreadLocal REMOTE_ADDRESS = + new ThreadLocal(); + + private @Context HttpServletRequest request; + private String method; + private String query; + private String reqPath; + + public RouterWebHdfsMethods(@Context HttpServletRequest request) { + super(request); + this.method = request.getMethod(); + this.query = request.getQueryString(); + this.reqPath = request.getServletPath(); + REMOTE_ADDRESS.set(JspHelper.getRemoteAddr(request)); + } + + @Override + protected void init(final UserGroupInformation ugi, + final DelegationParam delegation, + final UserParam username, final DoAsParam doAsUser, + final UriFsPathParam path, final HttpOpParam op, + final Param... parameters) { + super.init(ugi, delegation, username, doAsUser, path, op, parameters); + + REMOTE_ADDRESS.set(JspHelper.getRemoteAddr(request)); + } + + @Override + protected ClientProtocol getRpcClientProtocol() throws IOException { + final Router router = getRouter(); + final RouterRpcServer routerRpcServer = router.getRpcServer(); + if (routerRpcServer == null) { + throw new RetriableException("Router is in startup mode"); + } + return routerRpcServer; + } + + private void reset() { + REMOTE_ADDRESS.set(null); + } + + @Override + protected String getRemoteAddr() { + return REMOTE_ADDRESS.get(); + } + + @Override + protected void queueExternalCall(ExternalCall call) + throws IOException, InterruptedException { + getRouter().getRpcServer().getServer().queueCall(call); + } + + private Router getRouter() { + return (Router)getContext().getAttribute("name.node"); + } + + private static RouterRpcServer getRPCServer(final Router router) + throws IOException { + final RouterRpcServer routerRpcServer = router.getRpcServer(); + if (routerRpcServer == null) { + throw new RetriableException("Router is in startup mode"); + } + return routerRpcServer; + } + + @Override + protected Response put( + final UserGroupInformation ugi, + final DelegationParam delegation, + final UserParam username, + final DoAsParam doAsUser, + final String fullpath, + final PutOpParam op, + final DestinationParam destination, + final OwnerParam owner, + final GroupParam group, + final PermissionParam permission, + final UnmaskedPermissionParam unmaskedPermission, + final OverwriteParam overwrite, + final BufferSizeParam bufferSize, + final ReplicationParam replication, + final BlockSizeParam blockSize, + final ModificationTimeParam modificationTime, + final AccessTimeParam accessTime, + final RenameOptionSetParam renameOptions, + final CreateParentParam createParent, + final TokenArgumentParam delegationTokenArgument, + final AclPermissionParam aclPermission, + final XAttrNameParam xattrName, + final XAttrValueParam xattrValue, + final XAttrSetFlagParam xattrSetFlag, + final SnapshotNameParam snapshotName, + final OldSnapshotNameParam oldSnapshotName, + final ExcludeDatanodesParam exclDatanodes, + final CreateFlagParam createFlagParam, + final NoRedirectParam noredirectParam, + final StoragePolicyParam policyName + ) throws IOException, URISyntaxException { + + switch(op.getValue()) { + case CREATE: + { + final Router router = getRouter(); + final URI uri = redirectURI(router, fullpath); + if (!noredirectParam.getValue()) { + return Response.temporaryRedirect(uri) + .type(MediaType.APPLICATION_OCTET_STREAM).build(); + } else { + final String js = JsonUtil.toJsonString("Location", uri); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } + } + case MKDIRS: + case CREATESYMLINK: + case RENAME: + case SETREPLICATION: + case SETOWNER: + case SETPERMISSION: + case SETTIMES: + case RENEWDELEGATIONTOKEN: + case CANCELDELEGATIONTOKEN: + case MODIFYACLENTRIES: + case REMOVEACLENTRIES: + case REMOVEDEFAULTACL: + case REMOVEACL: + case SETACL: + case SETXATTR: + case REMOVEXATTR: + case ALLOWSNAPSHOT: + case CREATESNAPSHOT: + case RENAMESNAPSHOT: + case DISALLOWSNAPSHOT: + case SETSTORAGEPOLICY: + { + // Whitelist operations that can handled by NamenodeWebHdfsMethods + return super.put(ugi, delegation, username, doAsUser, fullpath, op, + destination, owner, group, permission, unmaskedPermission, + overwrite, bufferSize, replication, blockSize, modificationTime, + accessTime, renameOptions, createParent, delegationTokenArgument, + aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName, + oldSnapshotName, exclDatanodes, createFlagParam, noredirectParam, + policyName); + } + default: + throw new UnsupportedOperationException(op + " is not supported"); + } + } + + @Override + protected Response post( + final UserGroupInformation ugi, + final DelegationParam delegation, + final UserParam username, + final DoAsParam doAsUser, + final String fullpath, + final PostOpParam op, + final ConcatSourcesParam concatSrcs, + final BufferSizeParam bufferSize, + final ExcludeDatanodesParam excludeDatanodes, + final NewLengthParam newLength, + final NoRedirectParam noRedirectParam + ) throws IOException, URISyntaxException { + switch(op.getValue()) { + case APPEND: + { + final Router router = getRouter(); + final URI uri = redirectURI(router, ugi, delegation, username, + doAsUser, fullpath, op.getValue(), -1L, + excludeDatanodes.getValue(), bufferSize); + if (!noRedirectParam.getValue()) { + return Response.temporaryRedirect(uri) + .type(MediaType.APPLICATION_OCTET_STREAM).build(); + } else { + final String js = JsonUtil.toJsonString("Location", uri); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } + } + case CONCAT: + case TRUNCATE: + case UNSETSTORAGEPOLICY: + { + return super.post(ugi, delegation, username, doAsUser, fullpath, op, + concatSrcs, bufferSize, excludeDatanodes, newLength, + noRedirectParam); + } + default: + throw new UnsupportedOperationException(op + " is not supported"); + } + } + + @Override + protected Response get( + final UserGroupInformation ugi, + final DelegationParam delegation, + final UserParam username, + final DoAsParam doAsUser, + final String fullpath, + final GetOpParam op, + final OffsetParam offset, + final LengthParam length, + final RenewerParam renewer, + final BufferSizeParam bufferSize, + final List xattrNames, + final XAttrEncodingParam xattrEncoding, + final ExcludeDatanodesParam excludeDatanodes, + final FsActionParam fsAction, + final SnapshotNameParam snapshotName, + final OldSnapshotNameParam oldSnapshotName, + final TokenKindParam tokenKind, + final TokenServiceParam tokenService, + final NoRedirectParam noredirectParam, + final StartAfterParam startAfter + ) throws IOException, URISyntaxException { + try { + final Router router = getRouter(); + + switch (op.getValue()) { + case OPEN: + { + final URI uri = redirectURI(router, ugi, delegation, username, + doAsUser, fullpath, op.getValue(), offset.getValue(), + excludeDatanodes.getValue(), offset, length, bufferSize); + if (!noredirectParam.getValue()) { + return Response.temporaryRedirect(uri) + .type(MediaType.APPLICATION_OCTET_STREAM).build(); + } else { + final String js = JsonUtil.toJsonString("Location", uri); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } + } + case GETFILECHECKSUM: + { + final URI uri = redirectURI(router, ugi, delegation, username, + doAsUser, fullpath, op.getValue(), -1L, null); + if (!noredirectParam.getValue()) { + return Response.temporaryRedirect(uri) + .type(MediaType.APPLICATION_OCTET_STREAM).build(); + } else { + final String js = JsonUtil.toJsonString("Location", uri); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } + } + case GET_BLOCK_LOCATIONS: + case GETFILESTATUS: + case LISTSTATUS: + case GETCONTENTSUMMARY: + case GETHOMEDIRECTORY: + case GETACLSTATUS: + case GETXATTRS: + case LISTXATTRS: + case CHECKACCESS: + { + return super.get(ugi, delegation, username, doAsUser, fullpath, op, + offset, length, renewer, bufferSize, xattrNames, xattrEncoding, + excludeDatanodes, fsAction, snapshotName, oldSnapshotName, + tokenKind, tokenService, noredirectParam, startAfter); + } + default: + throw new UnsupportedOperationException(op + " is not supported"); + } + } finally { + reset(); + } + } + + /** + * Get the redirect URI from the Namenode responsible for a path. + * @param router Router to check. + * @param path Path to get location for. + * @return URI returned by the Namenode. + * @throws IOException If it cannot get the redirect URI. + */ + private URI redirectURI(final Router router, final String path) + throws IOException { + // Forward the request to the proper Namenode + final HttpURLConnection conn = forwardRequest(router, path); + try { + conn.setInstanceFollowRedirects(false); + conn.setDoOutput(true); + conn.connect(); + + // Read the reply from the Namenode + int responseCode = conn.getResponseCode(); + if (responseCode != HttpServletResponse.SC_TEMPORARY_REDIRECT) { + LOG.info("We expected a redirection from the Namenode, not {}", + responseCode); + return null; + } + + // Extract the redirect location and return it + String redirectLocation = conn.getHeaderField("Location"); + try { + // We modify the namenode location and the path + redirectLocation = redirectLocation + .replaceAll("(?<=[?&;])namenoderpcaddress=.*?(?=[&;])", + "namenoderpcaddress=" + router.getRouterId()) + .replaceAll("(?<=[/])webhdfs/v1/.*?(?=[?])", + "webhdfs/v1" + path); + return new URI(redirectLocation); + } catch (URISyntaxException e) { + LOG.error("Cannot parse redirect location {}", redirectLocation); + } + } finally { + if (conn != null) { + conn.disconnect(); + } + } + return null; + } + + /** + * Forwards a request to a subcluster. + * @param router Router to check. + * @param path Path in HDFS. + * @return Reply from the subcluster. + * @throws IOException + */ + private HttpURLConnection forwardRequest( + final Router router, final String path) throws IOException { + final Configuration conf = + (Configuration)getContext().getAttribute(JspHelper.CURRENT_CONF); + URLConnectionFactory connectionFactory = + URLConnectionFactory.newDefaultURLConnectionFactory(conf); + + // Find the namespace responsible for a path + final RouterRpcServer rpcServer = getRPCServer(router); + RemoteLocation createLoc = rpcServer.getCreateLocation(path); + String nsId = createLoc.getNameserviceId(); + String dest = createLoc.getDest(); + ActiveNamenodeResolver nnResolver = router.getNamenodeResolver(); + List namenodes = + nnResolver.getNamenodesForNameserviceId(nsId); + + // Go over the namenodes responsible for that namespace + for (FederationNamenodeContext namenode : namenodes) { + try { + // Generate the request for the namenode + String nnWebAddress = namenode.getWebAddress(); + String[] nnWebAddressSplit = nnWebAddress.split(":"); + String host = nnWebAddressSplit[0]; + int port = Integer.parseInt(nnWebAddressSplit[1]); + + // Avoid double-encoding here + query = URLDecoder.decode(query, "UTF-8"); + URI uri = new URI(getScheme(), null, host, port, + reqPath + dest, query, null); + URL url = uri.toURL(); + + // Send a request to the proper Namenode + final HttpURLConnection conn = + (HttpURLConnection)connectionFactory.openConnection(url); + conn.setRequestMethod(method); + + return conn; + } catch (Exception e) { + LOG.error("Cannot redirect request to {}", namenode, e); + } + } + return null; + } + + /** + * Get a URI to redirect an operation to. + * @param router Router to check. + * @param ugi User group information. + * @param delegation Delegation token. + * @param username User name. + * @param doAsUser Do as user. + * @param path Path to check. + * @param op Operation to perform. + * @param openOffset Offset for opening a file. + * @param excludeDatanodes Blocks to excluded. + * @param parameters Other parameters. + * @return Redirection URI. + * @throws URISyntaxException If it cannot parse the URI. + * @throws IOException If it cannot create the URI. + */ + private URI redirectURI(final Router router, final UserGroupInformation ugi, + final DelegationParam delegation, final UserParam username, + final DoAsParam doAsUser, final String path, final HttpOpParam.Op op, + final long openOffset, final String excludeDatanodes, + final Param... parameters) throws URISyntaxException, IOException { + final DatanodeInfo dn = + chooseDatanode(router, path, op, openOffset, excludeDatanodes); + + if (dn == null) { + throw new IOException("Failed to find datanode, suggest to check cluster" + + " health. excludeDatanodes=" + excludeDatanodes); + } + + final String delegationQuery; + if (!UserGroupInformation.isSecurityEnabled()) { + // security disabled + delegationQuery = Param.toSortedString("&", doAsUser, username); + } else if (delegation.getValue() != null) { + // client has provided a token + delegationQuery = "&" + delegation; + } else { + // generate a token + final Token t = generateDelegationToken( + router, ugi, request.getUserPrincipal().getName()); + delegationQuery = "&delegation=" + t.encodeToUrlString(); + } + + final String redirectQuery = op.toQueryString() + delegationQuery + + "&namenoderpcaddress=" + router.getRouterId() + + Param.toSortedString("&", parameters); + final String uripath = WebHdfsFileSystem.PATH_PREFIX + path; + + int port = "http".equals(getScheme()) ? dn.getInfoPort() : + dn.getInfoSecurePort(); + final URI uri = new URI(getScheme(), null, dn.getHostName(), port, uripath, + redirectQuery, null); + + if (LOG.isTraceEnabled()) { + LOG.trace("redirectURI={}", uri); + } + return uri; + } + + private DatanodeInfo chooseDatanode(final Router router, + final String path, final HttpOpParam.Op op, final long openOffset, + final String excludeDatanodes) throws IOException { + // We need to get the DNs as a privileged user + final RouterRpcServer rpcServer = getRPCServer(router); + UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); + + DatanodeInfo[] dns = loginUser.doAs( + new PrivilegedAction() { + @Override + public DatanodeInfo[] run() { + try { + return rpcServer.getDatanodeReport(DatanodeReportType.LIVE); + } catch (IOException e) { + LOG.error("Cannot get the datanodes from the RPC server", e); + return null; + } + } + }); + + HashSet excludes = new HashSet(); + if (excludeDatanodes != null) { + Collection collection = + getTrimmedStringCollection(excludeDatanodes); + for (DatanodeInfo dn : dns) { + if (collection.contains(dn.getName())) { + excludes.add(dn); + } + } + } + + if (op == GetOpParam.Op.OPEN || + op == PostOpParam.Op.APPEND || + op == GetOpParam.Op.GETFILECHECKSUM) { + // Choose a datanode containing a replica + final ClientProtocol cp = getRpcClientProtocol(); + final HdfsFileStatus status = cp.getFileInfo(path); + if (status == null) { + throw new FileNotFoundException("File " + path + " not found."); + } + final long len = status.getLen(); + if (op == GetOpParam.Op.OPEN) { + if (openOffset < 0L || (openOffset >= len && len > 0)) { + throw new IOException("Offset=" + openOffset + + " out of the range [0, " + len + "); " + op + ", path=" + path); + } + } + + if (len > 0) { + final long offset = op == GetOpParam.Op.OPEN ? openOffset : len - 1; + final LocatedBlocks locations = cp.getBlockLocations(path, offset, 1); + final int count = locations.locatedBlockCount(); + if (count > 0) { + LocatedBlock location0 = locations.get(0); + return bestNode(location0.getLocations(), excludes); + } + } + } + + return getRandomDatanode(dns, excludes); + } + + /** + * Get a random Datanode from a subcluster. + * @param dns Nodes to be chosen from. + * @param excludes Nodes to be excluded from. + * @return Random datanode from a particular subluster. + */ + private static DatanodeInfo getRandomDatanode( + final DatanodeInfo[] dns, final HashSet excludes) { + DatanodeInfo dn = null; + + if (dns == null) { + return dn; + } + + int numDNs = dns.length; + int availableNodes = 0; + if (excludes.isEmpty()) { + availableNodes = numDNs; + } else { + for (DatanodeInfo di : dns) { + if (!excludes.contains(di)) { + availableNodes++; + } + } + } + + // Return a random one from the list + if (availableNodes > 0) { + while (dn == null || excludes.contains(dn)) { + Random rnd = new Random(); + int idx = rnd.nextInt(numDNs); + dn = dns[idx]; + } + } + return dn; + } + + /** + * Generate the delegation tokens for this request. + * @param router Router. + * @param ugi User group information. + * @param renewer Who is asking for the renewal. + * @return The delegation tokens. + * @throws IOException If it cannot create the tokens. + */ + private Token generateDelegationToken( + final Router router, final UserGroupInformation ugi, + final String renewer) throws IOException { + throw new UnsupportedOperationException("TODO Generate token for ugi=" + + ugi + " request=" + request); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java new file mode 100644 index 0000000000000..2f1be52925da7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/RouterWebHDFSContract.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.contract.hdfs.HDFSContract; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.web.WebHdfsConstants; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES; + +/** + * The contract of Router-based Federated HDFS + * This changes its feature set from platform for platform -the default + * set is updated during initialization. + */ +public class RouterWebHDFSContract extends HDFSContract { + + public static final Logger LOG = + LoggerFactory.getLogger(WebHdfsFileSystem.class); + + public static final String CONTRACT_WEBHDFS_XML = "contract/webhdfs.xml"; + private static MiniRouterDFSCluster cluster; + + public RouterWebHDFSContract(Configuration conf) { + super(conf); + addConfResource(CONTRACT_WEBHDFS_XML); + } + + public static void createCluster() throws IOException { + try { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.addResource(CONTRACT_HDFS_XML); + conf.addResource(CONTRACT_WEBHDFS_XML); + + cluster = new MiniRouterDFSCluster(true, 2); + + // Start NNs and DNs and wait until ready + cluster.startCluster(); + + // Start routers with only an RPC service + cluster.startRouters(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + + // Setup the mount table + cluster.installMockLocations(); + + // Making one Namenodes active per nameservice + if (cluster.isHighAvailability()) { + for (String ns : cluster.getNameservices()) { + cluster.switchToActive(ns, NAMENODES[0]); + cluster.switchToStandby(ns, NAMENODES[1]); + } + } + } catch (Exception e) { + cluster = null; + throw new IOException("Cannot start federated cluster", e); + } + } + + public static void destroyCluster() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + public static MiniDFSCluster getCluster() { + return cluster.getCluster(); + } + + @Override + public FileSystem getTestFileSystem() throws IOException { + return getFileSystem(); + } + + public static FileSystem getFileSystem() throws IOException { + //assumes cluster is not null + Assert.assertNotNull("cluster not created", cluster); + + // Create a connection to WebHDFS + try { + RouterContext router = cluster.getRandomRouter(); + String uriStr = + WebHdfsConstants.WEBHDFS_SCHEME + "://" + router.getHttpAddress(); + URI uri = new URI(uriStr); + Configuration conf = new HdfsConfiguration(); + return FileSystem.get(uri, conf); + } catch (URISyntaxException e) { + LOG.error("Cannot create URI for the WebHDFS filesystem", e); + } + return null; + } + + @Override + public String getScheme() { + return WebHdfsConstants.WEBHDFS_SCHEME; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractAppend.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractAppend.java new file mode 100644 index 0000000000000..40278c204b149 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractAppend.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test append operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractAppend + extends AbstractContractAppendTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractConcat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractConcat.java new file mode 100644 index 0000000000000..b82a8e10e6c79 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractConcat.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractContractConcatTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test concat operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractConcat + extends AbstractContractConcatTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + // perform a simple operation on the cluster to verify it is up + RouterWebHDFSContract.getFileSystem().getDefaultBlockSize(new Path("/")); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractCreate.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractCreate.java new file mode 100644 index 0000000000000..ff1c610220f83 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractCreate.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test create operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractCreate + extends AbstractContractCreateTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractDelete.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractDelete.java new file mode 100644 index 0000000000000..dede65234b64e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractDelete.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test delete operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractDelete + extends AbstractContractDeleteTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractMkdir.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractMkdir.java new file mode 100644 index 0000000000000..9db4114be7071 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractMkdir.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractMkdir extends AbstractContractMkdirTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractOpen.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractOpen.java new file mode 100644 index 0000000000000..f5517ddaaf136 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractOpen.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test open operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractOpen extends AbstractContractOpenTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } + + @Override + @Test + public void testOpenReadDir() throws Throwable { + // WebHDFS itself allows open read on directory, we may need to + // fix this first before make this test work + } + + @Override + @Test + public void testOpenReadDirWithChild() throws Throwable { + // WebHDFS itself allows open read on directory, we may need to + // fix this first before make this test work + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRename.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRename.java new file mode 100644 index 0000000000000..a426ae0f002f4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRename.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test rename operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractRename + extends AbstractContractRenameTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java new file mode 100644 index 0000000000000..0128b3010d4bb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractRootDirectory.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractRootDirectory extends + AbstractContractRootDirectoryTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } + + @Override + public void testListEmptyRootDirectory() throws IOException { + // It doesn't apply because we still have the mount points here + } + + @Override + public void testRmEmptyRootDirNonRecursive() throws IOException { + // It doesn't apply because we still have the mount points here + } + + @Override + public void testRecursiveRootListing() throws IOException { + // It doesn't apply because we still have the mount points here + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractSeek.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractSeek.java new file mode 100644 index 0000000000000..5fbbc9b1e5a31 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/TestRouterWebHDFSContractSeek.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.router.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test seek operations on a Router WebHDFS FS. + */ +public class TestRouterWebHDFSContractSeek extends AbstractContractSeekTest { + + @BeforeClass + public static void createCluster() throws IOException { + RouterWebHDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + RouterWebHDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RouterWebHDFSContract(conf); + } + + @Override + public void testNegativeSeek() throws Throwable { + System.out.println("Not supported"); + } + + @Override + public void testSeekReadClosedFile() throws Throwable { + System.out.println("Not supported"); + } + + @Override + public void testSeekPastEndOfFileThenReseekAndRead() throws Throwable { + System.out.println("Not supported"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/package-info.java new file mode 100644 index 0000000000000..3411cf082fb52 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/fs/contract/router/web/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Test the WebHDFS contract. + */ +package org.apache.hadoop.fs.contract.router.web; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index df9f038156c7b..c49f90a497e4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -181,6 +181,11 @@ public FileContext getFileContext() { return this.fileContext; } + public String getHttpAddress() { + InetSocketAddress httpAddress = router.getHttpServerAddress(); + return NetUtils.getHostPortString(httpAddress); + } + public void initRouter() throws URISyntaxException { // Store the bound points for the router interfaces InetSocketAddress rpcAddress = router.getRpcServerAddress(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/webhdfs.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/webhdfs.xml new file mode 100644 index 0000000000000..f9b7d9435d26d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/resources/contract/webhdfs.xml @@ -0,0 +1,26 @@ + + + + + + fs.contract.supports-strict-exceptions + false + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index e7e5f512868f8..861afae5c7435 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -76,7 +76,9 @@ public class NameNodeHttpServer { this.bindAddress = bindAddress; } - private void initWebHdfs(Configuration conf) throws IOException { + public static void initWebHdfs(Configuration conf, String hostname, + HttpServer2 httpServer2, String jerseyResourcePackage) + throws IOException { // set user pattern based on configuration file UserParam.setUserPattern(conf.get( HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, @@ -92,8 +94,8 @@ private void initWebHdfs(Configuration conf) throws IOException { final String name = className; final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; - Map params = getAuthFilterParams(conf); - HttpServer2.defineFilter(httpServer.getWebAppContext(), name, className, + Map params = getAuthFilterParams(conf, hostname); + HttpServer2.defineFilter(httpServer2.getWebAppContext(), name, className, params, new String[] { pathSpec }); HttpServer2.LOG.info("Added filter '" + name + "' (class=" + className + ")"); @@ -104,13 +106,14 @@ private void initWebHdfs(Configuration conf) throws IOException { Map restCsrfParams = RestCsrfPreventionFilter .getFilterParams(conf, "dfs.webhdfs.rest-csrf."); String restCsrfClassName = RestCsrfPreventionFilter.class.getName(); - HttpServer2.defineFilter(httpServer.getWebAppContext(), restCsrfClassName, - restCsrfClassName, restCsrfParams, new String[] {pathSpec}); + HttpServer2.defineFilter(httpServer2.getWebAppContext(), + restCsrfClassName, restCsrfClassName, restCsrfParams, + new String[] {pathSpec}); } // add webhdfs packages - httpServer.addJerseyResourcePackage(NamenodeWebHdfsMethods.class - .getPackage().getName() + ";" + Param.class.getPackage().getName(), + httpServer2.addJerseyResourcePackage( + jerseyResourcePackage + ";" + Param.class.getPackage().getName(), pathSpec); } @@ -165,7 +168,8 @@ void start() throws IOException { datanodeSslPort.getPort()); } - initWebHdfs(conf); + initWebHdfs(conf, bindAddress.getHostName(), httpServer, + NamenodeWebHdfsMethods.class.getPackage().getName()); httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn); httpServer.setAttribute(JspHelper.CURRENT_CONF, conf); @@ -186,8 +190,8 @@ void start() throws IOException { } } - private Map getAuthFilterParams(Configuration conf) - throws IOException { + private static Map getAuthFilterParams(Configuration conf, + String hostname) throws IOException { Map params = new HashMap(); // Select configs beginning with 'dfs.web.authentication.' Iterator> iterator = conf.iterator(); @@ -203,8 +207,7 @@ private Map getAuthFilterParams(Configuration conf) params .put( DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, - SecurityUtil.getServerPrincipal(principalInConf, - bindAddress.getHostName())); + SecurityUtil.getServerPrincipal(principalInConf, hostname)); } else if (UserGroupInformation.isSecurityEnabled()) { HttpServer2.LOG.error( "WebHDFS and security are enabled, but configuration property '" + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 9f91f4263d5cd..a8ab798299d1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -143,7 +143,7 @@ public NamenodeWebHdfsMethods(@Context HttpServletRequest request) { Boolean.valueOf(request.getHeader(WebHdfsFileSystem.EZ_HEADER)); } - private void init(final UserGroupInformation ugi, + protected void init(final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, final DoAsParam doAsUser, final UriFsPathParam path, final HttpOpParam op, @@ -184,6 +184,14 @@ protected ClientProtocol getRpcClientProtocol() throws IOException { return cp; } + protected String getScheme() { + return scheme; + } + + protected ServletContext getContext() { + return context; + } + private T doAs(final UserGroupInformation ugi, final PrivilegedExceptionAction action) throws IOException, InterruptedException { @@ -206,7 +214,7 @@ public String getProtocol() { } @Override public String getHostAddress() { - return remoteAddr; + return getRemoteAddr(); } @Override public InetAddress getHostInetAddress() { @@ -217,8 +225,8 @@ public InetAddress getHostInetAddress() { } } }; - final NameNode namenode = (NameNode)context.getAttribute("name.node"); - namenode.queueExternalCall(call); + + queueExternalCall(call); T result = null; try { result = call.get(); @@ -235,6 +243,16 @@ public InetAddress getHostInetAddress() { return result; } + protected String getRemoteAddr() { + return remoteAddr; + } + + protected void queueExternalCall(ExternalCall call) + throws IOException, InterruptedException { + final NameNode namenode = (NameNode)context.getAttribute("name.node"); + namenode.queueExternalCall(call); + } + @VisibleForTesting static DatanodeInfo chooseDatanode(final NameNode namenode, final String path, final HttpOpParam.Op op, final long openOffset, @@ -306,7 +324,7 @@ static DatanodeInfo chooseDatanode(final NameNode namenode, * sorted based on availability and network distances, thus it is sufficient * to return the first element of the node here. */ - private static DatanodeInfo bestNode(DatanodeInfo[] nodes, + protected static DatanodeInfo bestNode(DatanodeInfo[] nodes, HashSet excludes) throws IOException { for (DatanodeInfo dn: nodes) { if (false == dn.isDecommissioned() && false == excludes.contains(dn)) { @@ -470,7 +488,7 @@ public Response putRoot( /** Validate all required params. */ @SuppressWarnings("rawtypes") - private void validateOpParams(HttpOpParam op, Param... params) { + protected void validateOpParams(HttpOpParam op, Param... params) { for (Param param : params) { if (param.getValue() == null || param.getValueString() == null || param .getValueString().isEmpty()) { @@ -570,7 +588,7 @@ public Response run() throws IOException, URISyntaxException { }); } - private Response put( + protected Response put( final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, @@ -602,14 +620,13 @@ private Response put( final NoRedirectParam noredirectParam, final StoragePolicyParam policyName ) throws IOException, URISyntaxException { - final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); - final NameNode namenode = (NameNode)context.getAttribute("name.node"); final ClientProtocol cp = getRpcClientProtocol(); switch(op.getValue()) { case CREATE: { + final NameNode namenode = (NameNode)context.getAttribute("name.node"); final URI uri = redirectURI(null, namenode, ugi, delegation, username, doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf), exclDatanodes.getValue(), permission, unmaskedPermission, @@ -840,7 +857,7 @@ public Response run() throws IOException, URISyntaxException { }); } - private Response post( + protected Response post( final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, @@ -1014,7 +1031,7 @@ private static String encodeFeInfo(FileEncryptionInfo feInfo) { return encodedValue; } - private Response get( + protected Response get( final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, @@ -1344,7 +1361,7 @@ public Response run() throws IOException { }); } - private Response delete( + protected Response delete( final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, From 98d7a5aaef2bbef46e0e7b6c876490f9235c59f5 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Fri, 23 Mar 2018 11:43:09 -0700 Subject: [PATCH 014/512] Update releasenotes and changelogs for 3.0.1 release --- .../markdown/release/3.0.1/CHANGES.3.0.1.md | 241 ++++++++++++++++++ .../release/3.0.1/RELEASENOTES.3.0.1.md | 54 ++++ 2 files changed, 295 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/CHANGES.3.0.1.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/RELEASENOTES.3.0.1.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/CHANGES.3.0.1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/CHANGES.3.0.1.md new file mode 100644 index 0000000000000..d24a8f4c3e948 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/CHANGES.3.0.1.md @@ -0,0 +1,241 @@ + + +# Apache Hadoop Changelog + +## Release 3.0.1 - 2018-03-16 + +### INCOMPATIBLE CHANGES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-12990](https://issues.apache.org/jira/browse/HDFS-12990) | Change default NameNode RPC port back to 8020 | Critical | namenode | Xiao Chen | Xiao Chen | + + +### IMPORTANT ISSUES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13083](https://issues.apache.org/jira/browse/HDFS-13083) | RBF: Fix doc error setting up client | Major | federation | tartarus | tartarus | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-14872](https://issues.apache.org/jira/browse/HADOOP-14872) | CryptoInputStream should implement unbuffer | Major | fs, security | John Zhuge | John Zhuge | +| [YARN-7414](https://issues.apache.org/jira/browse/YARN-7414) | FairScheduler#getAppWeight() should be moved into FSAppAttempt#getWeight() | Minor | fairscheduler | Daniel Templeton | Soumabrata Chakraborty | +| [HADOOP-15023](https://issues.apache.org/jira/browse/HADOOP-15023) | ValueQueue should also validate (lowWatermark \* numValues) \> 0 on construction | Minor | . | Xiao Chen | Xiao Chen | +| [HDFS-12814](https://issues.apache.org/jira/browse/HDFS-12814) | Add blockId when warning slow mirror/disk in BlockReceiver | Trivial | hdfs | Jiandan Yang | Jiandan Yang | +| [YARN-7524](https://issues.apache.org/jira/browse/YARN-7524) | Remove unused FairSchedulerEventLog | Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService | Major | log-aggregation | Jonathan Eagles | Jonathan Eagles | +| [YARN-7611](https://issues.apache.org/jira/browse/YARN-7611) | Node manager web UI should display container type in containers page | Major | nodemanager, webapp | Weiwei Yang | Weiwei Yang | +| [YARN-6483](https://issues.apache.org/jira/browse/YARN-6483) | Add nodes transitioning to DECOMMISSIONING state to the list of updated nodes returned to the AM | Major | resourcemanager | Juan Rodríguez Hortalá | Juan Rodríguez Hortalá | +| [HADOOP-15056](https://issues.apache.org/jira/browse/HADOOP-15056) | Fix TestUnbuffer#testUnbufferException failure | Minor | test | Jack Bearden | Jack Bearden | +| [HADOOP-15012](https://issues.apache.org/jira/browse/HADOOP-15012) | Add readahead, dropbehind, and unbuffer to StreamCapabilities | Major | fs | John Zhuge | John Zhuge | +| [HADOOP-15104](https://issues.apache.org/jira/browse/HADOOP-15104) | AliyunOSS: change the default value of max error retry | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-12910](https://issues.apache.org/jira/browse/HDFS-12910) | Secure Datanode Starter should log the port when it fails to bind | Minor | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-12819](https://issues.apache.org/jira/browse/HDFS-12819) | Setting/Unsetting EC policy shows warning if the directory is not empty | Minor | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-12927](https://issues.apache.org/jira/browse/HDFS-12927) | Update erasure coding doc to address unsupported APIs | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-5418](https://issues.apache.org/jira/browse/YARN-5418) | When partial log aggregation is enabled, display the list of aggregated files on the container log page | Major | . | Siddharth Seth | Xuan Gong | +| [HDFS-12818](https://issues.apache.org/jira/browse/HDFS-12818) | Support multiple storages in DataNodeCluster / SimulatedFSDataset | Minor | datanode, test | Erik Krogen | Erik Krogen | +| [HDFS-9023](https://issues.apache.org/jira/browse/HDFS-9023) | When NN is not able to identify DN for replication, reason behind it can be logged | Critical | hdfs-client, namenode | Surendra Singh Lilhore | Xiao Chen | +| [HDFS-11847](https://issues.apache.org/jira/browse/HDFS-11847) | Enhance dfsadmin listOpenFiles command to list files blocking datanode decommissioning | Major | hdfs | Manoj Govindassamy | Manoj Govindassamy | +| [YARN-7678](https://issues.apache.org/jira/browse/YARN-7678) | Ability to enable logging of container memory stats | Major | nodemanager | Jim Brennan | Jim Brennan | +| [HDFS-11848](https://issues.apache.org/jira/browse/HDFS-11848) | Enhance dfsadmin listOpenFiles command to list files under a given path | Major | . | Manoj Govindassamy | Yiqun Lin | +| [HDFS-12945](https://issues.apache.org/jira/browse/HDFS-12945) | Switch to ClientProtocol instead of NamenodeProtocols in NamenodeWebHdfsMethods | Minor | . | Wei Yan | Wei Yan | +| [YARN-7590](https://issues.apache.org/jira/browse/YARN-7590) | Improve container-executor validation check | Major | security, yarn | Eric Yang | Eric Yang | +| [MAPREDUCE-6984](https://issues.apache.org/jira/browse/MAPREDUCE-6984) | MR AM to clean up temporary files from previous attempt in case of no recovery | Major | applicationmaster | Gergo Repas | Gergo Repas | +| [HADOOP-15185](https://issues.apache.org/jira/browse/HADOOP-15185) | Update adls connector to use the current version of ADLS SDK | Major | fs/adl | Atul Sikaria | Atul Sikaria | +| [HADOOP-15189](https://issues.apache.org/jira/browse/HADOOP-15189) | backport HADOOP-15039 to branch-2 and branch-3 | Blocker | . | Genmao Yu | Genmao Yu | +| [HADOOP-15186](https://issues.apache.org/jira/browse/HADOOP-15186) | Allow Azure Data Lake SDK dependency version to be set on the command line | Major | build, fs/adl | Vishwajeet Dusane | Vishwajeet Dusane | +| [HDFS-13092](https://issues.apache.org/jira/browse/HDFS-13092) | Reduce verbosity for ThrottledAsyncChecker.java:schedule | Minor | datanode | Mukul Kumar Singh | Mukul Kumar Singh | +| [HDFS-13062](https://issues.apache.org/jira/browse/HDFS-13062) | Provide support for JN to use separate journal disk per namespace | Major | . | Bharat Viswanadham | Bharat Viswanadham | +| [HADOOP-15212](https://issues.apache.org/jira/browse/HADOOP-15212) | Add independent secret manager method for logging expired tokens | Major | security | Daryn Sharp | Daryn Sharp | +| [YARN-7728](https://issues.apache.org/jira/browse/YARN-7728) | Expose container preemptions related information in Capacity Scheduler queue metrics | Major | . | Eric Payne | Eric Payne | +| [MAPREDUCE-7048](https://issues.apache.org/jira/browse/MAPREDUCE-7048) | Uber AM can crash due to unknown task in statusUpdate | Major | mr-am | Peter Bacsko | Peter Bacsko | +| [HADOOP-15204](https://issues.apache.org/jira/browse/HADOOP-15204) | Add Configuration API for parsing storage sizes | Minor | conf | Anu Engineer | Anu Engineer | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-7361](https://issues.apache.org/jira/browse/YARN-7361) | Improve the docker container runtime documentation | Major | . | Shane Kumpf | Shane Kumpf | +| [YARN-7489](https://issues.apache.org/jira/browse/YARN-7489) | ConcurrentModificationException in RMAppImpl#getRMAppMetrics | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-7525](https://issues.apache.org/jira/browse/YARN-7525) | Incorrect query parameters in cluster nodes REST API document | Minor | documentation | Tao Yang | Tao Yang | +| [HADOOP-15046](https://issues.apache.org/jira/browse/HADOOP-15046) | Document Apache Hadoop does not support Java 9 in BUILDING.txt | Major | documentation | Akira Ajisaka | Hanisha Koneru | +| [YARN-7531](https://issues.apache.org/jira/browse/YARN-7531) | ResourceRequest.equal does not check ExecutionTypeRequest.enforceExecutionType() | Major | api | Haibo Chen | Haibo Chen | +| [YARN-7513](https://issues.apache.org/jira/browse/YARN-7513) | Remove scheduler lock in FSAppAttempt.getWeight() | Minor | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-7390](https://issues.apache.org/jira/browse/YARN-7390) | All reservation related test cases failed when TestYarnClient runs against Fair Scheduler. | Major | fairscheduler, reservation system | Yufei Gu | Yufei Gu | +| [YARN-7363](https://issues.apache.org/jira/browse/YARN-7363) | ContainerLocalizer doesn't have a valid log4j config when using LinuxContainerExecutor | Major | nodemanager | Yufei Gu | Yufei Gu | +| [HDFS-12832](https://issues.apache.org/jira/browse/HDFS-12832) | INode.getFullPathName may throw ArrayIndexOutOfBoundsException lead to NameNode exit | Critical | namenode | DENG FEI | Konstantin Shvachko | +| [HADOOP-15042](https://issues.apache.org/jira/browse/HADOOP-15042) | Azure PageBlobInputStream.skip() can return negative value when numberOfPagesRemaining is 0 | Minor | fs/azure | Rajesh Balamohan | Rajesh Balamohan | +| [YARN-7558](https://issues.apache.org/jira/browse/YARN-7558) | "yarn logs" command fails to get logs for running containers if UI authentication is enabled. | Critical | . | Namit Maheshwari | Xuan Gong | +| [HDFS-12638](https://issues.apache.org/jira/browse/HDFS-12638) | Delete copy-on-truncate block along with the original block, when deleting a file being truncated | Blocker | hdfs | Jiandan Yang | Konstantin Shvachko | +| [HDFS-12836](https://issues.apache.org/jira/browse/HDFS-12836) | startTxId could be greater than endTxId when tailing in-progress edit log | Major | hdfs | Chao Sun | Chao Sun | +| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events | Major | mr-am | Jason Lowe | Peter Bacsko | +| [YARN-7589](https://issues.apache.org/jira/browse/YARN-7589) | TestPBImplRecords fails with NullPointerException | Major | . | Jason Lowe | Daniel Templeton | +| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer | Major | nodemanager | Jason Lowe | Jim Brennan | +| [HADOOP-15058](https://issues.apache.org/jira/browse/HADOOP-15058) | create-release site build outputs dummy shaded jars due to skipShade | Blocker | . | Andrew Wang | Andrew Wang | +| [HADOOP-14985](https://issues.apache.org/jira/browse/HADOOP-14985) | Remove subversion related code from VersionInfoMojo.java | Minor | build | Akira Ajisaka | Ajay Kumar | +| [HADOOP-15098](https://issues.apache.org/jira/browse/HADOOP-15098) | TestClusterTopology#testChooseRandom fails intermittently | Major | test | Zsolt Venczel | Zsolt Venczel | +| [HDFS-12891](https://issues.apache.org/jira/browse/HDFS-12891) | Do not invalidate blocks if toInvalidate is empty | Major | . | Zsolt Venczel | Zsolt Venczel | +| [YARN-7647](https://issues.apache.org/jira/browse/YARN-7647) | NM print inappropriate error log when node-labels is enabled | Minor | . | Yang Wang | Yang Wang | +| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers | Major | namenode | Daryn Sharp | Rushabh S Shah | +| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Lowe | Ajay Kumar | +| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes | Major | nodemanager | Jason Lowe | Jim Brennan | +| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Lowe | Jim Brennan | +| [YARN-7629](https://issues.apache.org/jira/browse/YARN-7629) | TestContainerLaunch# fails after YARN-7381 | Major | . | Jason Lowe | Jason Lowe | +| [YARN-7664](https://issues.apache.org/jira/browse/YARN-7664) | Several javadoc errors | Blocker | . | Sean Mackrory | Sean Mackrory | +| [HADOOP-15123](https://issues.apache.org/jira/browse/HADOOP-15123) | KDiag tries to load krb5.conf from KRB5CCNAME instead of KRB5\_CONFIG | Minor | security | Vipin Rathor | Vipin Rathor | +| [YARN-7661](https://issues.apache.org/jira/browse/YARN-7661) | NodeManager metrics return wrong value after update node resource | Major | . | Yang Wang | Yang Wang | +| [HDFS-12347](https://issues.apache.org/jira/browse/HDFS-12347) | TestBalancerRPCDelay#testBalancerRPCDelay fails very frequently | Critical | test | Xiao Chen | Bharat Viswanadham | +| [HDFS-12930](https://issues.apache.org/jira/browse/HDFS-12930) | Remove the extra space in HdfsImageViewer.md | Trivial | documentation | Yiqun Lin | Rahul Pathak | +| [YARN-7662](https://issues.apache.org/jira/browse/YARN-7662) | [Atsv2] Define new set of configurations for reader and collectors to bind. | Major | . | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-12845](https://issues.apache.org/jira/browse/HDFS-12845) | JournalNode Test failures | Major | journal-node | Bharat Viswanadham | Bharat Viswanadham | +| [YARN-7466](https://issues.apache.org/jira/browse/YARN-7466) | ResourceRequest has a different default for allocationRequestId than Container | Major | . | Chandni Singh | Chandni Singh | +| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 | Major | . | Rohith Sharma K S | Sunil G | +| [HDFS-12938](https://issues.apache.org/jira/browse/HDFS-12938) | TestErasureCodigCLI testAll failing consistently. | Major | erasure-coding, hdfs | Rushabh S Shah | Ajay Kumar | +| [YARN-7542](https://issues.apache.org/jira/browse/YARN-7542) | Fix issue that causes some Running Opportunistic Containers to be recovered as PAUSED | Major | . | Arun Suresh | Sampada Dehankar | +| [HDFS-12915](https://issues.apache.org/jira/browse/HDFS-12915) | Fix findbugs warning in INodeFile$HeaderFormat.getBlockLayoutRedundancy | Major | namenode | Wei-Chiu Chuang | Chris Douglas | +| [HADOOP-15122](https://issues.apache.org/jira/browse/HADOOP-15122) | Lock down version of doxia-module-markdown plugin | Blocker | . | Elek, Marton | Elek, Marton | +| [HADOOP-15143](https://issues.apache.org/jira/browse/HADOOP-15143) | NPE due to Invalid KerberosTicket in UGI | Major | . | Jitendra Nath Pandey | Mukul Kumar Singh | +| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications | Blocker | resourcemanager | Charan Hebri | Sunil G | +| [MAPREDUCE-7028](https://issues.apache.org/jira/browse/MAPREDUCE-7028) | Concurrent task progress updates causing NPE in Application Master | Blocker | mr-am | Gergo Repas | Gergo Repas | +| [YARN-7602](https://issues.apache.org/jira/browse/YARN-7602) | NM should reference the singleton JvmMetrics instance | Major | nodemanager | Haibo Chen | Haibo Chen | +| [HDFS-12913](https://issues.apache.org/jira/browse/HDFS-12913) | TestDNFencingWithReplication.testFencingStress fix mini cluster not yet active issue | Major | . | Zsolt Venczel | Zsolt Venczel | +| [HDFS-12860](https://issues.apache.org/jira/browse/HDFS-12860) | StripedBlockUtil#getRangesInternalBlocks throws exception for the block group size larger than 2GB | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-7619](https://issues.apache.org/jira/browse/YARN-7619) | Max AM Resource value in Capacity Scheduler UI has to be refreshed for every user | Major | capacity scheduler, yarn | Eric Payne | Eric Payne | +| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call | Major | webapp | Sunil G | Sunil G | +| [HDFS-12985](https://issues.apache.org/jira/browse/HDFS-12985) | NameNode crashes during restart after an OpenForWrite file present in the Snapshot got deleted | Major | hdfs | Manoj Govindassamy | Manoj Govindassamy | +| [YARN-7508](https://issues.apache.org/jira/browse/YARN-7508) | NPE in FiCaSchedulerApp when debug log enabled in async-scheduling mode | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-7663](https://issues.apache.org/jira/browse/YARN-7663) | RMAppImpl:Invalid event: START at KILLED | Minor | resourcemanager | lujie | lujie | +| [YARN-6948](https://issues.apache.org/jira/browse/YARN-6948) | Invalid event: ATTEMPT\_ADDED at FINAL\_SAVING | Minor | yarn | lujie | lujie | +| [HDFS-12994](https://issues.apache.org/jira/browse/HDFS-12994) | TestReconstructStripedFile.testNNSendsErasureCodingTasks fails due to socket timeout | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-7665](https://issues.apache.org/jira/browse/YARN-7665) | Allow FS scheduler state dump to be turned on/off separately from FS debug log | Major | . | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HADOOP-15060](https://issues.apache.org/jira/browse/HADOOP-15060) | TestShellBasedUnixGroupsMapping.testFiniteGroupResolutionTime flaky | Major | . | Miklos Szegedi | Miklos Szegedi | +| [YARN-7735](https://issues.apache.org/jira/browse/YARN-7735) | Fix typo in YARN documentation | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-7727](https://issues.apache.org/jira/browse/YARN-7727) | Incorrect log levels in few logs with QueuePriorityContainerCandidateSelector | Minor | yarn | Prabhu Joseph | Prabhu Joseph | +| [HDFS-11915](https://issues.apache.org/jira/browse/HDFS-11915) | Sync rbw dir on the first hsync() to avoid file lost on power failure | Critical | . | Kanaka Kumar Avvaru | Vinayakumar B | +| [YARN-7705](https://issues.apache.org/jira/browse/YARN-7705) | Create the container log directory with correct sticky bit in C code | Major | nodemanager | Yufei Gu | Yufei Gu | +| [YARN-7479](https://issues.apache.org/jira/browse/YARN-7479) | TestContainerManagerSecurity.testContainerManager[Simple] flaky in trunk | Major | test | Botong Huang | Akira Ajisaka | +| [HDFS-13004](https://issues.apache.org/jira/browse/HDFS-13004) | TestLeaseRecoveryStriped#testLeaseRecovery is failing when safeLength is 0MB or larger than the test file | Major | hdfs | Zsolt Venczel | Zsolt Venczel | +| [HDFS-9049](https://issues.apache.org/jira/browse/HDFS-9049) | Make Datanode Netty reverse proxy port to be configurable | Major | datanode | Vinayakumar B | Vinayakumar B | +| [YARN-7758](https://issues.apache.org/jira/browse/YARN-7758) | Add an additional check to the validity of container and application ids passed to container-executor | Major | nodemanager | Miklos Szegedi | Yufei Gu | +| [HADOOP-15150](https://issues.apache.org/jira/browse/HADOOP-15150) | in FsShell, UGI params should be overidden through env vars(-D arg) | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-15166](https://issues.apache.org/jira/browse/HADOOP-15166) | CLI MiniCluster fails with ClassNotFoundException o.a.h.yarn.server.timelineservice.collector.TimelineCollectorManager | Major | . | Gera Shegalov | Gera Shegalov | +| [HDFS-13039](https://issues.apache.org/jira/browse/HDFS-13039) | StripedBlockReader#createBlockReader leaks socket on IOException | Critical | datanode, erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HADOOP-15181](https://issues.apache.org/jira/browse/HADOOP-15181) | Typo in SecureMode.md | Trivial | documentation | Masahiro Tanaka | Masahiro Tanaka | +| [YARN-7796](https://issues.apache.org/jira/browse/YARN-7796) | Container-executor fails with segfault on certain OS configurations | Major | nodemanager | Gergo Repas | Gergo Repas | +| [YARN-7806](https://issues.apache.org/jira/browse/YARN-7806) | Distributed Shell should use timeline async api's | Major | distributed-shell | Sumana Sathish | Rohith Sharma K S | +| [MAPREDUCE-7015](https://issues.apache.org/jira/browse/MAPREDUCE-7015) | Possible race condition in JHS if the job is not loaded | Major | jobhistoryserver | Peter Bacsko | Peter Bacsko | +| [YARN-7737](https://issues.apache.org/jira/browse/YARN-7737) | prelaunch.err file not found exception on container failure | Major | . | Jonathan Hung | Keqiu Hu | +| [HDFS-13063](https://issues.apache.org/jira/browse/HDFS-13063) | Fix the incorrect spelling in HDFSHighAvailabilityWithQJM.md | Trivial | documentation | Jianfei Jiang | Jianfei Jiang | +| [YARN-7102](https://issues.apache.org/jira/browse/YARN-7102) | NM heartbeat stuck when responseId overflows MAX\_INT | Critical | . | Botong Huang | Botong Huang | +| [MAPREDUCE-7041](https://issues.apache.org/jira/browse/MAPREDUCE-7041) | MR should not try to clean up at first job attempt | Major | . | Takanobu Asanuma | Gergo Repas | +| [HDFS-13054](https://issues.apache.org/jira/browse/HDFS-13054) | Handling PathIsNotEmptyDirectoryException in DFSClient delete call | Major | hdfs-client | Nanda kumar | Nanda kumar | +| [MAPREDUCE-7020](https://issues.apache.org/jira/browse/MAPREDUCE-7020) | Task timeout in uber mode can crash AM | Major | mr-am | Akira Ajisaka | Peter Bacsko | +| [YARN-7765](https://issues.apache.org/jira/browse/YARN-7765) | [Atsv2] GSSException: No valid credentials provided - Failed to find any Kerberos tgt thrown by Timelinev2Client & HBaseClient in NM | Blocker | . | Sumana Sathish | Rohith Sharma K S | +| [HDFS-13065](https://issues.apache.org/jira/browse/HDFS-13065) | TestErasureCodingMultipleRacks#testSkewedRack3 is failing | Major | hdfs | Gabor Bota | Gabor Bota | +| [HDFS-12974](https://issues.apache.org/jira/browse/HDFS-12974) | Exception message is not printed when creating an encryption zone fails with AuthorizationException | Minor | encryption | fang zhenyi | fang zhenyi | +| [YARN-7698](https://issues.apache.org/jira/browse/YARN-7698) | A misleading variable's name in ApplicationAttemptEventDispatcher | Minor | resourcemanager | Jinjiang Ling | Jinjiang Ling | +| [YARN-7790](https://issues.apache.org/jira/browse/YARN-7790) | Improve Capacity Scheduler Async Scheduling to better handle node failures | Critical | . | Sumana Sathish | Wangda Tan | +| [HDFS-12528](https://issues.apache.org/jira/browse/HDFS-12528) | Add an option to not disable short-circuit reads on failures | Major | hdfs-client, performance | Andre Araujo | Xiao Chen | +| [HDFS-12897](https://issues.apache.org/jira/browse/HDFS-12897) | getErasureCodingPolicy should handle .snapshot dir better | Major | erasure-coding, hdfs, snapshots | Harshakiran Reddy | LiXin Ge | +| [MAPREDUCE-7033](https://issues.apache.org/jira/browse/MAPREDUCE-7033) | Map outputs implicitly rely on permissive umask for shuffle | Critical | mrv2 | Jason Lowe | Jason Lowe | +| [HDFS-12942](https://issues.apache.org/jira/browse/HDFS-12942) | Synchronization issue in FSDataSetImpl#moveBlock | Major | . | Ajay Kumar | Ajay Kumar | +| [HDFS-13048](https://issues.apache.org/jira/browse/HDFS-13048) | LowRedundancyReplicatedBlocks metric can be negative | Major | metrics | Akira Ajisaka | Akira Ajisaka | +| [HDFS-13100](https://issues.apache.org/jira/browse/HDFS-13100) | Handle IllegalArgumentException when GETSERVERDEFAULTS is not implemented in webhdfs. | Critical | hdfs, webhdfs | Yongjun Zhang | Yongjun Zhang | +| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error | Major | test | Jason Lowe | Botong Huang | +| [YARN-7801](https://issues.apache.org/jira/browse/YARN-7801) | AmFilterInitializer should addFilter after fill all parameters | Critical | . | Sumana Sathish | Wangda Tan | +| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch | Major | . | Billie Rinaldi | Jason Lowe | +| [YARN-7873](https://issues.apache.org/jira/browse/YARN-7873) | Revert YARN-6078 | Blocker | . | Billie Rinaldi | Billie Rinaldi | +| [HDFS-13115](https://issues.apache.org/jira/browse/HDFS-13115) | In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted | Major | . | Yongjun Zhang | Yongjun Zhang | +| [HDFS-12935](https://issues.apache.org/jira/browse/HDFS-12935) | Get ambiguous result for DFSAdmin command in HA mode when only one namenode is up | Major | tools | Jianfei Jiang | Jianfei Jiang | +| [HDFS-13120](https://issues.apache.org/jira/browse/HDFS-13120) | Snapshot diff could be corrupted after concat | Major | namenode, snapshots | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. | Major | namenode | He Xiaoqiao | He Xiaoqiao | +| [HDFS-8693](https://issues.apache.org/jira/browse/HDFS-8693) | refreshNamenodes does not support adding a new standby to a running DN | Critical | datanode, ha | Jian Fang | Ajith S | +| [MAPREDUCE-7052](https://issues.apache.org/jira/browse/MAPREDUCE-7052) | TestFixedLengthInputFormat#testFormatCompressedIn is flaky | Major | client, test | Peter Bacsko | Peter Bacsko | +| [HDFS-13112](https://issues.apache.org/jira/browse/HDFS-13112) | Token expiration edits may cause log corruption or deadlock | Critical | namenode | Daryn Sharp | Daryn Sharp | +| [YARN-7937](https://issues.apache.org/jira/browse/YARN-7937) | Fix http method name in Cluster Application Timeout Update API example request | Minor | docs, documentation | Charan Hebri | Charan Hebri | +| [HADOOP-10571](https://issues.apache.org/jira/browse/HADOOP-10571) | Use Log.\*(Object, Throwable) overload to log exceptions | Major | . | Arpit Agarwal | Andras Bokor | +| [HDFS-13164](https://issues.apache.org/jira/browse/HDFS-13164) | File not closed if streamer fail with DSQuotaExceededException | Major | hdfs-client | Xiao Chen | Xiao Chen | +| [HDFS-13244](https://issues.apache.org/jira/browse/HDFS-13244) | Add stack, conf, metrics links to utilities dropdown in NN webUI | Major | . | Bharat Viswanadham | Bharat Viswanadham | +| [YARN-8022](https://issues.apache.org/jira/browse/YARN-8022) | ResourceManager UI cluster/app/\ page fails to render | Blocker | webapp | Tarun Parimi | Tarun Parimi | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [MAPREDUCE-7011](https://issues.apache.org/jira/browse/MAPREDUCE-7011) | TestClientDistributedCacheManager::testDetermineCacheVisibilities assumes all parent dirs set other exec | Trivial | . | Chris Douglas | Chris Douglas | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-14993](https://issues.apache.org/jira/browse/HADOOP-14993) | AliyunOSS: Override listFiles and listLocatedStatus | Major | fs/oss | Genmao Yu | Genmao Yu | +| [YARN-6953](https://issues.apache.org/jira/browse/YARN-6953) | Clean up ResourceUtils.setMinimumAllocationForMandatoryResources() and setMaximumAllocationForMandatoryResources() | Minor | resourcemanager | Daniel Templeton | Manikandan R | +| [HDFS-12801](https://issues.apache.org/jira/browse/HDFS-12801) | RBF: Set MountTableResolver as default file resolver | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-15024](https://issues.apache.org/jira/browse/HADOOP-15024) | AliyunOSS: support user agent configuration and include that & Hadoop version information to oss server | Major | fs, fs/oss | SammiChen | SammiChen | +| [HDFS-12858](https://issues.apache.org/jira/browse/HDFS-12858) | RBF: Add router admin commands usage in HDFS commands reference doc | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [HDFS-12835](https://issues.apache.org/jira/browse/HDFS-12835) | RBF: Fix Javadoc parameter errors | Minor | . | Wei Yan | Wei Yan | +| [YARN-6907](https://issues.apache.org/jira/browse/YARN-6907) | Node information page in the old web UI should report resource types | Major | resourcemanager | Daniel Templeton | Gergely Novák | +| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. | Major | encryption, kms, webhdfs | Rushabh S Shah | Rushabh S Shah | +| [YARN-7610](https://issues.apache.org/jira/browse/YARN-7610) | Extend Distributed Shell to support launching job with opportunistic containers | Major | applications/distributed-shell | Weiwei Yang | Weiwei Yang | +| [HDFS-12875](https://issues.apache.org/jira/browse/HDFS-12875) | RBF: Complete logic for -readonly option of dfsrouteradmin add command | Major | . | Yiqun Lin | Íñigo Goiri | +| [YARN-7383](https://issues.apache.org/jira/browse/YARN-7383) | Node resource is not parsed correctly for resource names containing dot | Major | nodemanager, resourcemanager | Jonathan Hung | Gergely Novák | +| [YARN-7119](https://issues.apache.org/jira/browse/YARN-7119) | Support multiple resource types in rmadmin updateNodeResource command | Major | nodemanager, resourcemanager | Daniel Templeton | Manikandan R | +| [YARN-7617](https://issues.apache.org/jira/browse/YARN-7617) | Add a flag in distributed shell to automatically PROMOTE opportunistic containers to guaranteed once they are started | Minor | applications/distributed-shell | Weiwei Yang | Weiwei Yang | +| [HDFS-12937](https://issues.apache.org/jira/browse/HDFS-12937) | RBF: Add more unit tests for router admin commands | Major | test | Yiqun Lin | Yiqun Lin | +| [YARN-7032](https://issues.apache.org/jira/browse/YARN-7032) | [ATSv2] NPE while starting hbase co-processor when HBase authorization is enabled. | Critical | . | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-14965](https://issues.apache.org/jira/browse/HADOOP-14965) | s3a input stream "normal" fadvise mode to be adaptive | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-15086](https://issues.apache.org/jira/browse/HADOOP-15086) | NativeAzureFileSystem file rename is not atomic | Major | fs/azure | Shixiong Zhu | Thomas Marquardt | +| [HDFS-12988](https://issues.apache.org/jira/browse/HDFS-12988) | RBF: Mount table entries not properly updated in the local cache | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-7716](https://issues.apache.org/jira/browse/YARN-7716) | metricsTimeStart and metricsTimeEnd should be all lower case in the doc | Major | timelinereader | Haibo Chen | Haibo Chen | +| [HDFS-12802](https://issues.apache.org/jira/browse/HDFS-12802) | RBF: Control MountTableResolver cache size | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-12919](https://issues.apache.org/jira/browse/HDFS-12919) | RBF: Support erasure coding methods in RouterRpcServer | Critical | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-6736](https://issues.apache.org/jira/browse/YARN-6736) | Consider writing to both ats v1 & v2 from RM for smoother upgrades | Major | timelineserver | Vrushali C | Aaron Gresch | +| [HADOOP-15027](https://issues.apache.org/jira/browse/HADOOP-15027) | AliyunOSS: Support multi-thread pre-read to improve sequential read from Hadoop to Aliyun OSS performance | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-13028](https://issues.apache.org/jira/browse/HDFS-13028) | RBF: Fix spurious TestRouterRpc#testProxyGetStats | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-5094](https://issues.apache.org/jira/browse/YARN-5094) | some YARN container events have timestamp of -1 | Critical | timelineserver | Sangjin Lee | Haibo Chen | +| [YARN-7782](https://issues.apache.org/jira/browse/YARN-7782) | Enable user re-mapping for Docker containers in yarn-default.xml | Blocker | security, yarn | Eric Yang | Eric Yang | +| [HDFS-12772](https://issues.apache.org/jira/browse/HDFS-12772) | RBF: Federation Router State State Store internal API | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13042](https://issues.apache.org/jira/browse/HDFS-13042) | RBF: Heartbeat Router State | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13049](https://issues.apache.org/jira/browse/HDFS-13049) | RBF: Inconsistent Router OPTS config in branch-2 and branch-3 | Minor | . | Wei Yan | Wei Yan | +| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. | Major | encryption, kms, webhdfs | Rushabh S Shah | Rushabh S Shah | +| [HDFS-13044](https://issues.apache.org/jira/browse/HDFS-13044) | RBF: Add a safe mode for the Router | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13043](https://issues.apache.org/jira/browse/HDFS-13043) | RBF: Expose the state of the Routers in the federation | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-12997](https://issues.apache.org/jira/browse/HDFS-12997) | Move logging to slf4j in BlockPoolSliceStorage and Storage | Major | . | Ajay Kumar | Ajay Kumar | +| [HDFS-13068](https://issues.apache.org/jira/browse/HDFS-13068) | RBF: Add router admin option to manage safe mode | Major | . | Íñigo Goiri | Yiqun Lin | +| [HADOOP-15247](https://issues.apache.org/jira/browse/HADOOP-15247) | Move commons-net up to 3.6 | Minor | fs | Steve Loughran | Steve Loughran | +| [HADOOP-15090](https://issues.apache.org/jira/browse/HADOOP-15090) | Add ADL troubleshooting doc | Major | documentation, fs/adl | Steve Loughran | Steve Loughran | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15149](https://issues.apache.org/jira/browse/HADOOP-15149) | CryptoOutputStream should implement StreamCapabilities | Major | fs | Mike Drob | Xiao Chen | +| [YARN-7691](https://issues.apache.org/jira/browse/YARN-7691) | Add Unit Tests for ContainersLauncher | Major | . | Sampada Dehankar | Sampada Dehankar | +| [HADOOP-15177](https://issues.apache.org/jira/browse/HADOOP-15177) | Update the release year to 2018 | Blocker | build | Akira Ajisaka | Bharat Viswanadham | +| [HADOOP-15197](https://issues.apache.org/jira/browse/HADOOP-15197) | Remove tomcat from the Hadoop-auth test bundle | Major | . | Xiao Chen | Xiao Chen | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/RELEASENOTES.3.0.1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/RELEASENOTES.3.0.1.md new file mode 100644 index 0000000000000..e9663bcd2efdf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.1/RELEASENOTES.3.0.1.md @@ -0,0 +1,54 @@ + + +# Apache Hadoop 3.0.1 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HADOOP-15027](https://issues.apache.org/jira/browse/HADOOP-15027) | *Major* | **AliyunOSS: Support multi-thread pre-read to improve sequential read from Hadoop to Aliyun OSS performance** + +Support multi-thread pre-read in AliyunOSSInputStream to improve the sequential read performance from Hadoop to Aliyun OSS. + + +--- + +* [HDFS-12528](https://issues.apache.org/jira/browse/HDFS-12528) | *Major* | **Add an option to not disable short-circuit reads on failures** + +Added an option to not disables short-circuit reads on failures, by setting dfs.domain.socket.disable.interval.seconds to 0. + + +--- + +* [HDFS-13083](https://issues.apache.org/jira/browse/HDFS-13083) | *Major* | **RBF: Fix doc error setting up client** + +Fix the document error of setting up HFDS Router Federation + + +--- + +* [HDFS-12990](https://issues.apache.org/jira/browse/HDFS-12990) | *Critical* | **Change default NameNode RPC port back to 8020** + +HDFS NameNode default RPC port is changed back to 8020. The only official release that has this differently is 3.0.0, which used 9820. + +It is recommended for 2.x users to upgrade to 3.0.1+, to reduce the burden on changing default NN RPC port. + + + From 4859cd7cc936d5fcf115a2f1cb06fe45a742ff5d Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Fri, 23 Mar 2018 11:48:36 -0700 Subject: [PATCH 015/512] Update 3.0.1 jdiff file and jdiff stable api version --- .../jdiff/Apache_Hadoop_HDFS_3.0.1.xml | 324 ++++++++++++++++++ hadoop-project-dist/pom.xml | 2 +- 2 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.1.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.1.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.1.xml new file mode 100644 index 0000000000000..91c8a6ba3122f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.1.xml @@ -0,0 +1,324 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

    + +

    The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 9118a71ca5136..7b714ee741b2b 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -145,7 +145,7 @@ false - 3.0.0 + 3.0.1 -unstable From c98dd3bf18f6551f5be912ed36264a21cf8d42d9 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Fri, 23 Mar 2018 16:44:44 -0700 Subject: [PATCH 016/512] YARN-8070. Yarn Service API site doc broken due to unwanted character in YarnServiceAPI.md (Gour Saha via wangda) Change-Id: I22428b2f128d16e79ebbdeaf32869566963d0c6f (cherry picked from commit 24f75e097a67ae90d2ff382bb2f2559caa02f32f) --- .../src/site/markdown/yarn-service/YarnServiceAPI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index 1e18b9969f53d..63e0af56fa840 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -1,4 +1,4 @@ -# + + + + + + + + + + + + + + + + + + +
    namevaluedescription
    + + +     +     From 1176a128d632925749db0a81a719fd8141c3cbb6 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 27 Mar 2018 13:33:41 -0700 Subject: [PATCH 025/512] HADOOP-15312. Undocumented KeyProvider configuration keys. Contributed by LiXin Ge. (cherry picked from commit 3fe41c65d84843f817a4f9ef8999dbf862db6674) --- .../src/main/resources/core-default.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 90743008298e5..49bfa2881c98b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2494,6 +2494,25 @@ + + hadoop.security.key.default.bitlength + 128 + + The length (bits) of keys we want the KeyProvider to produce. Key length + defines the upper-bound on an algorithm's security, ideally, it would + coincide with the lower-bound on an algorithm's security. + + + + + hadoop.security.key.default.cipher + AES/CTR/NoPadding + + This indicates the algorithm that be used by KeyProvider for generating + key, and will be converted to CipherSuite when creating encryption zone. + + + fs.har.impl.disable.cache true From 85eebf1bebc7b191dcd692395f77903257cd85c4 Mon Sep 17 00:00:00 2001 From: Subru Krishnan Date: Tue, 27 Mar 2018 17:39:46 -0700 Subject: [PATCH 026/512] YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru). (cherry picked from commit 2a2ef15caf791f30c471526c1b74e68803f0c405) --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 9 ++++++--- .../src/main/resources/yarn-default.xml | 9 +++++++++ .../failover/FederationRMFailoverProxyProvider.java | 11 ++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b76f45798e9d0..6390afc1da422 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3087,15 +3087,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS = FEDERATION_PREFIX + "cache-ttl.secs"; + // 5 minutes + public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60; + + public static final String FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = + FEDERATION_PREFIX + "flush-cache-for-rm-addr"; + public static final boolean DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = true; public static final String FEDERATION_REGISTRY_BASE_KEY = FEDERATION_PREFIX + "registry.base-dir"; public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY = "yarnfederation/"; - // 5 minutes - public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60; - public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS = FEDERATION_PREFIX + "state-store.heartbeat-interval-secs"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 114ba4bc3f2dd..2eba8df670ad2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2924,6 +2924,15 @@ 300 + + + Whether to flush FederationStateStoreFacade cache to get subcluster info + when FederationRMFailoverProxyProvider is performing failover. + + yarn.federation.flush-cache-for-rm-addr + true + + The registry base directory for federation. yarn.federation.registry.base-dir diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java index c631208b78390..b72b19935945f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java @@ -64,7 +64,8 @@ public class FederationRMFailoverProxyProvider private FederationStateStoreFacade facade; private SubClusterId subClusterId; private UserGroupInformation originalUser; - private boolean federationFailoverEnabled = false; + private boolean federationFailoverEnabled; + private boolean flushFacadeCacheForYarnRMAddr; @Override public void init(Configuration configuration, RMProxy proxy, @@ -75,13 +76,16 @@ public void init(Configuration configuration, RMProxy proxy, String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID); Preconditions.checkNotNull(clusterId, "Missing RM ClusterId"); this.subClusterId = SubClusterId.newInstance(clusterId); - this.facade = facade.getInstance(); + this.facade = FederationStateStoreFacade.getInstance(); if (configuration instanceof YarnConfiguration) { this.conf = (YarnConfiguration) configuration; } federationFailoverEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED); + flushFacadeCacheForYarnRMAddr = + conf.getBoolean(YarnConfiguration.FEDERATION_FLUSh_CACHE_FOR_RM_ADDR, + YarnConfiguration.DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR); conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, @@ -119,7 +123,8 @@ private T getProxyInternal(boolean isFailover) { try { LOG.info("Failing over to the ResourceManager for SubClusterId: {}", subClusterId); - subClusterInfo = facade.getSubCluster(subClusterId, isFailover); + subClusterInfo = facade.getSubCluster(subClusterId, + this.flushFacadeCacheForYarnRMAddr && isFailover); // updating the conf with the refreshed RM addresses as proxy // creations are based out of conf updateRMAddress(subClusterInfo); From 6c904388f401d3ca1641050fa4ff9996dff54aa4 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 28 Mar 2018 08:03:18 +0530 Subject: [PATCH 027/512] YARN-8075. DShell does not fail when we ask more GPUs than available even though AM throws 'InvalidResourceRequestException'. Contributed by Wangda Tan. (cherry picked from commit d1e378d02bdaec2f078ccc89698e7959ffcc71b3) --- .../yarn/applications/distributedshell/ApplicationMaster.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 5c107750f16e2..75f4073a92c79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -1147,7 +1147,6 @@ public float getProgress() { public void onError(Throwable e) { LOG.error("Error in RMCallbackHandler: ", e); done = true; - amRMClient.stop(); } } From 99766992e6e2995046621827a64272afede62b99 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 28 Mar 2018 11:00:08 +0800 Subject: [PATCH 028/512] HDFS-13347. RBF: Cache datanode reports. Contributed by Inigo Goiri. (cherry picked from commit a71656c1c1bf6c680f1382a76ddcac870061f320) --- .../federation/metrics/FederationMetrics.java | 2 +- .../metrics/NamenodeBeanMetrics.java | 68 ++++++++++++++++++- .../hdfs/server/federation/router/Router.java | 13 ++++ .../router/RouterMetricsService.java | 9 +++ .../federation/router/RouterRpcServer.java | 16 ++--- .../federation/router/TestRouterRpc.java | 41 ++++++++++- 6 files changed, 135 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java index a80c3be2e0626..a99a26a309da7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java @@ -434,7 +434,7 @@ public String getNodeUsage() { try { RouterRpcServer rpcServer = this.router.getRpcServer(); DatanodeInfo[] live = rpcServer.getDatanodeReport( - DatanodeReportType.LIVE, TIME_OUT); + DatanodeReportType.LIVE, false, TIME_OUT); if (live.length > 0) { float totalDfsUsed = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java index c4e5b5bcdc025..e8c6c8220399d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java @@ -25,6 +25,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -32,6 +34,7 @@ import javax.management.ObjectName; import javax.management.StandardMBean; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -39,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; +import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; import org.apache.hadoop.hdfs.server.federation.router.Router; import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; import org.apache.hadoop.hdfs.server.federation.store.MembershipStore; @@ -56,6 +60,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + /** * Expose the Namenode metrics as the Router was one. */ @@ -65,6 +73,22 @@ public class NamenodeBeanMetrics private static final Logger LOG = LoggerFactory.getLogger(NamenodeBeanMetrics.class); + /** Prevent holding the page from loading too long. */ + private static final String DN_REPORT_TIME_OUT = + RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.time-out"; + /** We only wait for 1 second. */ + private static final long DN_REPORT_TIME_OUT_DEFAULT = + TimeUnit.SECONDS.toMillis(1); + + /** Time to cache the DN information. */ + public static final String DN_REPORT_CACHE_EXPIRE = + RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire"; + /** We cache the DN information for 10 seconds by default. */ + public static final long DN_REPORT_CACHE_EXPIRE_DEFAULT = + TimeUnit.SECONDS.toMillis(10); + + + /** Instance of the Router being monitored. */ private final Router router; /** FSNamesystem bean. */ @@ -76,6 +100,11 @@ public class NamenodeBeanMetrics /** NameNodeStatus bean. */ private ObjectName nnStatusBeanName; + /** Timeout to get the DN report. */ + private final long dnReportTimeOut; + /** DN type -> full DN report in JSON. */ + private final LoadingCache dnCache; + public NamenodeBeanMetrics(Router router) { this.router = router; @@ -114,6 +143,23 @@ public NamenodeBeanMetrics(Router router) { } catch (NotCompliantMBeanException e) { throw new RuntimeException("Bad NameNodeStatus MBean setup", e); } + + // Initialize the cache for the DN reports + Configuration conf = router.getConfig(); + this.dnReportTimeOut = conf.getTimeDuration( + DN_REPORT_TIME_OUT, DN_REPORT_TIME_OUT_DEFAULT, TimeUnit.MILLISECONDS); + long dnCacheExpire = conf.getTimeDuration( + DN_REPORT_CACHE_EXPIRE, + DN_REPORT_CACHE_EXPIRE_DEFAULT, TimeUnit.MILLISECONDS); + this.dnCache = CacheBuilder.newBuilder() + .expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS) + .build( + new CacheLoader() { + @Override + public String load(DatanodeReportType type) throws Exception { + return getNodesImpl(type); + } + }); } /** @@ -298,17 +344,33 @@ public String getDecomNodes() { return this.getNodes(DatanodeReportType.DECOMMISSIONING); } + /** + * Get all the nodes in the federation from a particular type. Getting this + * information is expensive and we use a cache. + * @param type Type of the datanodes to check. + * @return JSON with the nodes. + */ + private String getNodes(final DatanodeReportType type) { + try { + return this.dnCache.get(type); + } catch (ExecutionException e) { + LOG.error("Cannot get the DN storage report for {}", type, e); + } + // If we cannot get the report, return empty JSON + return "{}"; + } + /** * Get all the nodes in the federation from a particular type. - * TODO this is expensive, we may want to cache it. * @param type Type of the datanodes to check. * @return JSON with the nodes. */ - private String getNodes(DatanodeReportType type) { + private String getNodesImpl(final DatanodeReportType type) { final Map> info = new HashMap<>(); try { RouterRpcServer rpcServer = this.router.getRpcServer(); - DatanodeInfo[] datanodes = rpcServer.getDatanodeReport(type); + DatanodeInfo[] datanodes = + rpcServer.getDatanodeReport(type, false, dnReportTimeOut); for (DatanodeInfo node : datanodes) { Map innerinfo = new HashMap<>(); innerinfo.put("infoAddr", node.getInfoAddr()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java index 38f5d4ff968fc..df2a4486f9b64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics; +import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; import org.apache.hadoop.hdfs.server.federation.store.RouterStore; @@ -552,6 +553,18 @@ public FederationMetrics getMetrics() { return null; } + /** + * Get the Namenode metrics. + * + * @return Namenode metrics. + */ + public NamenodeBeanMetrics getNamenodeMetrics() { + if (this.metrics != null) { + return this.metrics.getNamenodeMetrics(); + } + return null; + } + /** * Get the subcluster resolver for files. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java index f4debce3aae43..1887ed6bce2f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java @@ -94,6 +94,15 @@ public FederationMetrics getFederationMetrics() { return this.federationMetrics; } + /** + * Get the Namenode metrics. + * + * @return Namenode metrics. + */ + public NamenodeBeanMetrics getNamenodeMetrics() { + return this.nnMetrics; + } + /** * Get the JVM metrics for the Router. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index eaa39515fb722..383fd776bbf0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -1237,18 +1237,20 @@ public long[] getStats() throws IOException { public DatanodeInfo[] getDatanodeReport(DatanodeReportType type) throws IOException { checkOperation(OperationCategory.UNCHECKED); - return getDatanodeReport(type, 0); + return getDatanodeReport(type, true, 0); } /** * Get the datanode report with a timeout. * @param type Type of the datanode. + * @param requireResponse If we require all the namespaces to report. * @param timeOutMs Time out for the reply in milliseconds. * @return List of datanodes. * @throws IOException If it cannot get the report. */ public DatanodeInfo[] getDatanodeReport( - DatanodeReportType type, long timeOutMs) throws IOException { + DatanodeReportType type, boolean requireResponse, long timeOutMs) + throws IOException { checkOperation(OperationCategory.UNCHECKED); Map datanodesMap = new LinkedHashMap<>(); @@ -1257,8 +1259,8 @@ public DatanodeInfo[] getDatanodeReport( Set nss = namenodeResolver.getNamespaces(); Map results = - rpcClient.invokeConcurrent( - nss, method, true, false, timeOutMs, DatanodeInfo[].class); + rpcClient.invokeConcurrent(nss, method, requireResponse, false, + timeOutMs, DatanodeInfo[].class); for (Entry entry : results.entrySet()) { FederationNamespaceInfo ns = entry.getKey(); @@ -1278,9 +1280,7 @@ public DatanodeInfo[] getDatanodeReport( } // Map -> Array Collection datanodes = datanodesMap.values(); - DatanodeInfo[] combinedData = new DatanodeInfo[datanodes.size()]; - combinedData = datanodes.toArray(combinedData); - return combinedData; + return toArray(datanodes, DatanodeInfo.class); } @Override // ClientProtocol @@ -2295,7 +2295,7 @@ protected static T[] merge( * @param clazz Class of the values. * @return Array with the values in set. */ - private static T[] toArray(Set set, Class clazz) { + private static T[] toArray(Collection set, Class clazz) { @SuppressWarnings("unchecked") T[] combinedData = (T[]) Array.newInstance(clazz, set.size()); combinedData = set.toArray(combinedData); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index e8341a2a06eae..50148806c9407 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -27,6 +27,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -45,6 +46,7 @@ import java.util.Random; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.CryptoProtocolVersion; @@ -71,10 +73,11 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.io.EnumSetWritable; @@ -83,6 +86,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.test.GenericTestUtils; +import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -150,7 +154,14 @@ public static void globalSetUp() throws Exception { cluster.startCluster(); // Start routers with only an RPC service - cluster.addRouterOverrides((new RouterConfigBuilder()).rpc().build()); + Configuration routerConf = new RouterConfigBuilder() + .metrics() + .rpc() + .build(); + // We decrease the DN cache times to make the test faster + routerConf.setTimeDuration( + NamenodeBeanMetrics.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS); + cluster.addRouterOverrides(routerConf); cluster.startRouters(); // Register and verify all NNs with all routers @@ -1032,6 +1043,32 @@ public void testErasureCoding() throws IOException { assertEquals(statsNamenode.toString(), statsRouter.toString()); } + @Test + public void testNamenodeMetrics() throws Exception { + final NamenodeBeanMetrics metrics = + router.getRouter().getNamenodeMetrics(); + final String jsonString0 = metrics.getLiveNodes(); + + // We should have 12 nodes in total + JSONObject jsonObject = new JSONObject(jsonString0); + assertEquals(12, jsonObject.names().length()); + + // We should be caching this information + String jsonString1 = metrics.getLiveNodes(); + assertEquals(jsonString0, jsonString1); + + // We wait until the cached value is updated + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return !jsonString0.equals(metrics.getLiveNodes()); + } + }, 500, 5 * 1000); + + // The cache should be updated now + assertNotEquals(jsonString0, metrics.getLiveNodes()); + } + /** * Check the erasure coding policies in the Router and the Namenode. * @return The erasure coding policies. From a2b4daab55f3ba841835a2d3440f451f0ced9a99 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Wed, 28 Mar 2018 08:47:31 -0700 Subject: [PATCH 029/512] YARN-6629. NPE occurred when container allocation proposal is applied but its resource requests are removed before. (Tao Yang via wangda) Change-Id: I805880f90b3f6798ec96ed8e8e75755f390a9ad5 (cherry picked from commit 47f711eebca315804c80012eea5f31275ac25518) --- .../scheduler/capacity/CapacityScheduler.java | 4 +- .../common/fica/FiCaSchedulerApp.java | 16 ++++-- .../capacity/TestCapacityScheduler.java | 53 +++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index ddab0c1bd82fd..d33e0c2a01c6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2789,8 +2789,8 @@ public boolean tryCommit(Resource cluster, ResourceCommitRequest r, // proposal might be outdated if AM failover just finished // and proposal queue was not be consumed in time if (app != null && attemptId.equals(app.getApplicationAttemptId())) { - if (app.accept(cluster, request, updatePending)) { - app.apply(cluster, request, updatePending); + if (app.accept(cluster, request, updatePending) + && app.apply(cluster, request, updatePending)) { LOG.info("Allocation proposal accepted"); isSuccess = true; } else{ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index f3da0a36f0b20..32b2cad0ddf89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -489,7 +489,7 @@ public boolean accept(Resource cluster, return accepted; } - public void apply(Resource cluster, ResourceCommitRequest request, boolean updatePending) { boolean reReservation = false; @@ -502,8 +502,16 @@ public void apply(Resource cluster, ResourceCommitRequest schedulerContainer = allocation.getAllocatedOrReservedContainer(); - RMContainer rmContainer = schedulerContainer.getRmContainer(); + // Required sanity check - AM can call 'allocate' to update resource + // request without locking the scheduler, hence we need to check + if (updatePending && + getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey()) + <= 0) { + return false; + } + + RMContainer rmContainer = schedulerContainer.getRmContainer(); reReservation = (!schedulerContainer.isAllocated()) && (rmContainer.getState() == RMContainerState.RESERVED); @@ -545,7 +553,8 @@ public void apply(Resource cluster, ResourceCommitRequest() { + public Object answer(InvocationOnMock invocation) throws Exception { + // clear resource request before applying the proposal for container_2 + spyCs.allocate(app.getCurrentAppAttempt().getAppAttemptId(), + Arrays.asList(ResourceRequest.newInstance(priority, "*", + Resources.createResource(1 * GB), 0)), null, + Collections.emptyList(), null, null, + NULL_UPDATE_REQUESTS); + // trigger real apply which can raise NPE before YARN-6629 + try { + FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt( + app.getCurrentAppAttempt().getAppAttemptId()); + schedulerApp.apply((Resource) invocation.getArguments()[0], + (ResourceCommitRequest) invocation.getArguments()[1], + (Boolean) invocation.getArguments()[2]); + // the proposal of removed request should be rejected + Assert.assertEquals(1, schedulerApp.getLiveContainers().size()); + } catch (Throwable e) { + Assert.fail(); + } + return null; + } + }).when(spyCs).tryCommit(Mockito.any(Resource.class), + Mockito.any(ResourceCommitRequest.class), Mockito.anyBoolean()); + + // rm allocates container_2 to reproduce the process that can raise NPE + spyCs.allocate(app.getCurrentAppAttempt().getAppAttemptId(), + Arrays.asList(ResourceRequest.newInstance(priority, "*", + Resources.createResource(1 * GB), 1)), null, + Collections.emptyList(), null, null, NULL_UPDATE_REQUESTS); + spyCs.handle(new NodeUpdateSchedulerEvent( + spyCs.getNode(nm.getNodeId()).getRMNode())); + } } From ed49f59e1896a81c9ab4e3a744413f9f1bb37da0 Mon Sep 17 00:00:00 2001 From: Subru Krishnan Date: Wed, 28 Mar 2018 11:35:30 -0700 Subject: [PATCH 030/512] Revert "YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru)." This reverts commit 85eebf1bebc7b191dcd692395f77903257cd85c4. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 9 +++------ .../src/main/resources/yarn-default.xml | 9 --------- .../failover/FederationRMFailoverProxyProvider.java | 11 +++-------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6390afc1da422..b76f45798e9d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3087,18 +3087,15 @@ public static boolean isAclEnabled(Configuration conf) { public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS = FEDERATION_PREFIX + "cache-ttl.secs"; - // 5 minutes - public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60; - - public static final String FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = - FEDERATION_PREFIX + "flush-cache-for-rm-addr"; - public static final boolean DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = true; public static final String FEDERATION_REGISTRY_BASE_KEY = FEDERATION_PREFIX + "registry.base-dir"; public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY = "yarnfederation/"; + // 5 minutes + public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60; + public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS = FEDERATION_PREFIX + "state-store.heartbeat-interval-secs"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2eba8df670ad2..114ba4bc3f2dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2924,15 +2924,6 @@ 300 - - - Whether to flush FederationStateStoreFacade cache to get subcluster info - when FederationRMFailoverProxyProvider is performing failover. - - yarn.federation.flush-cache-for-rm-addr - true - - The registry base directory for federation. yarn.federation.registry.base-dir diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java index b72b19935945f..c631208b78390 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java @@ -64,8 +64,7 @@ public class FederationRMFailoverProxyProvider private FederationStateStoreFacade facade; private SubClusterId subClusterId; private UserGroupInformation originalUser; - private boolean federationFailoverEnabled; - private boolean flushFacadeCacheForYarnRMAddr; + private boolean federationFailoverEnabled = false; @Override public void init(Configuration configuration, RMProxy proxy, @@ -76,16 +75,13 @@ public void init(Configuration configuration, RMProxy proxy, String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID); Preconditions.checkNotNull(clusterId, "Missing RM ClusterId"); this.subClusterId = SubClusterId.newInstance(clusterId); - this.facade = FederationStateStoreFacade.getInstance(); + this.facade = facade.getInstance(); if (configuration instanceof YarnConfiguration) { this.conf = (YarnConfiguration) configuration; } federationFailoverEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED); - flushFacadeCacheForYarnRMAddr = - conf.getBoolean(YarnConfiguration.FEDERATION_FLUSh_CACHE_FOR_RM_ADDR, - YarnConfiguration.DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR); conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, @@ -123,8 +119,7 @@ private T getProxyInternal(boolean isFailover) { try { LOG.info("Failing over to the ResourceManager for SubClusterId: {}", subClusterId); - subClusterInfo = facade.getSubCluster(subClusterId, - this.flushFacadeCacheForYarnRMAddr && isFailover); + subClusterInfo = facade.getSubCluster(subClusterId, isFailover); // updating the conf with the refreshed RM addresses as proxy // creations are based out of conf updateRMAddress(subClusterInfo); From a0091ec4b3c28a468f95d8f066a769320937ad6f Mon Sep 17 00:00:00 2001 From: Subru Krishnan Date: Wed, 28 Mar 2018 11:33:19 -0700 Subject: [PATCH 031/512] YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru). (cherry picked from commit 09999d7e014fde717e8b122773b68664f4594106) --- .../hadoop/yarn/conf/YarnConfiguration.java | 9 ++- .../conf/TestYarnConfigurationFields.java | 2 + ...TestFederationRMFailoverProxyProvider.java | 81 ++++++++++++++++++- .../FederationRMFailoverProxyProvider.java | 11 ++- 4 files changed, 94 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b76f45798e9d0..5a2c1f91dd482 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3087,15 +3087,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS = FEDERATION_PREFIX + "cache-ttl.secs"; + // 5 minutes + public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60; + + public static final String FEDERATION_FLUSH_CACHE_FOR_RM_ADDR = + FEDERATION_PREFIX + "flush-cache-for-rm-addr"; + public static final boolean DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR = true; public static final String FEDERATION_REGISTRY_BASE_KEY = FEDERATION_PREFIX + "registry.base-dir"; public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY = "yarnfederation/"; - // 5 minutes - public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60; - public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS = FEDERATION_PREFIX + "state-store.heartbeat-interval-secs"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 9fe4f884899b0..f4d1ac0a25579 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -79,6 +79,8 @@ public void initializeMemberVariables() { .add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED); configurationPropsToSkipCompare .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS); + configurationPrefixToSkipCompare + .add(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR); configurationPropsToSkipCompare .add(YarnConfiguration.RM_EPOCH); configurationPropsToSkipCompare diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java index e3f91557ee46e..0a7ee3ff734c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; @@ -49,7 +50,11 @@ import org.junit.Test; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -61,12 +66,20 @@ public class TestFederationRMFailoverProxyProvider { private FederationStateStore stateStore; private final String dummyCapability = "cap"; + private GetClusterMetricsResponse threadResponse; + @Before public void setUp() throws IOException, YarnException { conf = new YarnConfiguration(); - stateStore = new MemoryFederationStateStore(); + + // Configure Facade cache to use a very long ttl + conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 60 * 60); + + stateStore = spy(new MemoryFederationStateStore()); stateStore.init(conf); FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + verify(stateStore, times(0)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); } @After @@ -75,12 +88,25 @@ public void tearDown() throws Exception { stateStore = null; } - @Test + @Test(timeout = 60000) public void testFederationRMFailoverProxyProvider() throws Exception { + testProxyProvider(true); + } + + @Test (timeout=60000) + public void testFederationRMFailoverProxyProviderWithoutFlushFacadeCache() + throws Exception { + testProxyProvider(false); + } + + private void testProxyProvider(boolean facadeFlushCache) throws Exception { final SubClusterId subClusterId = SubClusterId.newInstance("SC-1"); final MiniYARNCluster cluster = new MiniYARNCluster( "testFederationRMFailoverProxyProvider", 3, 0, 1, 1); + conf.setBoolean(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR, + facadeFlushCache); + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1"); @@ -104,10 +130,16 @@ public void testFederationRMFailoverProxyProvider() throws Exception { .createRMProxy(conf, ApplicationClientProtocol.class, subClusterId, UserGroupInformation.getCurrentUser()); + verify(stateStore, times(1)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); + // client will retry until the rm becomes active. GetClusterMetricsResponse response = client.getClusterMetrics(GetClusterMetricsRequest.newInstance()); + verify(stateStore, times(1)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); + // validate response checkResponse(response); @@ -118,7 +150,50 @@ public void testFederationRMFailoverProxyProvider() throws Exception { // Transition rm2 to active; makeRMActive(subClusterId, cluster, 1); - response = client.getClusterMetrics(GetClusterMetricsRequest.newInstance()); + + verify(stateStore, times(1)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); + + threadResponse = null; + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + // In non flush cache case, we will be hitting the cache with old RM + // address and keep failing before the cache is flushed + threadResponse = + client.getClusterMetrics(GetClusterMetricsRequest.newInstance()); + } catch (YarnException | IOException e) { + e.printStackTrace(); + } + } + }); + thread.start(); + + if (!facadeFlushCache) { + // Add a wait so that hopefully the thread has started hitting old cached + Thread.sleep(500); + + // Should still be hitting cache + verify(stateStore, times(1)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); + + // Force flush cache, so that it will pick up the new RM address + FederationStateStoreFacade.getInstance().getSubCluster(subClusterId, + true); + } + + // Wait for the thread to finish and grab result + thread.join(); + response = threadResponse; + + if (facadeFlushCache) { + verify(stateStore, atLeast(2)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); + } else { + verify(stateStore, times(2)) + .getSubClusters(any(GetSubClustersInfoRequest.class)); + } // validate response checkResponse(response); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java index c631208b78390..cf6d1ef5bd04d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java @@ -64,7 +64,8 @@ public class FederationRMFailoverProxyProvider private FederationStateStoreFacade facade; private SubClusterId subClusterId; private UserGroupInformation originalUser; - private boolean federationFailoverEnabled = false; + private boolean federationFailoverEnabled; + private boolean flushFacadeCacheForYarnRMAddr; @Override public void init(Configuration configuration, RMProxy proxy, @@ -75,13 +76,16 @@ public void init(Configuration configuration, RMProxy proxy, String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID); Preconditions.checkNotNull(clusterId, "Missing RM ClusterId"); this.subClusterId = SubClusterId.newInstance(clusterId); - this.facade = facade.getInstance(); + this.facade = FederationStateStoreFacade.getInstance(); if (configuration instanceof YarnConfiguration) { this.conf = (YarnConfiguration) configuration; } federationFailoverEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED); + flushFacadeCacheForYarnRMAddr = + conf.getBoolean(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR, + YarnConfiguration.DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR); conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, @@ -119,7 +123,8 @@ private T getProxyInternal(boolean isFailover) { try { LOG.info("Failing over to the ResourceManager for SubClusterId: {}", subClusterId); - subClusterInfo = facade.getSubCluster(subClusterId, isFailover); + subClusterInfo = facade.getSubCluster(subClusterId, + this.flushFacadeCacheForYarnRMAddr && isFailover); // updating the conf with the refreshed RM addresses as proxy // creations are based out of conf updateRMAddress(subClusterInfo); From 0db6b8c93cd964291707c3ba03bfc63bb65b710a Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Wed, 28 Mar 2018 11:41:13 -0700 Subject: [PATCH 032/512] YARN-7623. Fix the CapacityScheduler Queue configuration documentation. Contributed by Jonathan Hung. (cherry picked from commit 0b1c2b5fe1b5c225d208936ecb1d3e307a535ee6) --- .../src/site/markdown/CapacityScheduler.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 4ecc97a31c940..32162943f2812 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -415,14 +415,14 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw **Note:** This feature is in alpha phase and is subject to change. - | Property | Description | - |:---- |:---- | - | `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). | - | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. | - | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. | - | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. | - | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. | - | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. | +| Property | Description | +|:---- |:---- | +| `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). | +| `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. | +| `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. | +| `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. | +| `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. | +| `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. | **Note:** When enabling scheduler configuration mutations via `yarn.scheduler.configuration.store.class`, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file. From ac16e8f4d3957c64bc2afddb9e58bd2f511f6c6e Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Wed, 28 Mar 2018 11:58:59 -0700 Subject: [PATCH 033/512] HADOOP-15320. Remove customized getFileBlockLocations for hadoop-azure and hadoop-azure-datalake. Contributed by Shanyu Zhao (cherry picked from commit 081c3501885c543bb1f159929d456d1ba2e3650c) --- .../apache/hadoop/fs/adl/AdlFileSystem.java | 40 ----- .../fs/azure/NativeAzureFileSystem.java | 46 ------ ...stNativeAzureFileSystemBlockLocations.java | 141 ------------------ 3 files changed, 227 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 9f54a36c68579..aa6babea4d24b 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -46,7 +46,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.ContentSummary.Builder; import org.apache.hadoop.fs.CreateFlag; @@ -910,45 +909,6 @@ public long getBlockSize(Path f) throws IOException { return ADL_BLOCK_SIZE; } - @Override - public BlockLocation[] getFileBlockLocations(final FileStatus status, - final long offset, final long length) throws IOException { - if (status == null) { - return null; - } - - if ((offset < 0) || (length < 0)) { - throw new IllegalArgumentException("Invalid start or len parameter"); - } - - if (status.getLen() < offset) { - return new BlockLocation[0]; - } - - final String[] name = {"localhost"}; - final String[] host = {"localhost"}; - long blockSize = ADL_BLOCK_SIZE; - int numberOfLocations = - (int) (length / blockSize) + ((length % blockSize == 0) ? 0 : 1); - BlockLocation[] locations = new BlockLocation[numberOfLocations]; - for (int i = 0; i < locations.length; i++) { - long currentOffset = offset + (i * blockSize); - long currentLength = Math.min(blockSize, offset + length - currentOffset); - locations[i] = new BlockLocation(name, host, currentOffset, - currentLength); - } - - return locations; - } - - @Override - public BlockLocation[] getFileBlockLocations(final Path p, final long offset, - final long length) throws IOException { - // read ops incremented in getFileStatus - FileStatus fileStatus = getFileStatus(p); - return getFileBlockLocations(fileStatus, offset, length); - } - /** * Get replication. * diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 3d44b2045e3e4..e05327e4b318b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -52,7 +52,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BufferedFSInputStream; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -726,10 +725,6 @@ public String getScheme() { static final String AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE = "*"; - static final String AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME = - "fs.azure.block.location.impersonatedhost"; - private static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = - "localhost"; static final String AZURE_RINGBUFFER_CAPACITY_PROPERTY_NAME = "fs.azure.ring.buffer.capacity"; static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME = @@ -3469,47 +3464,6 @@ private void performStickyBitCheckForRenameOperation(Path srcPath, } } - /** - * Return an array containing hostnames, offset and size of - * portions of the given file. For WASB we'll just lie and give - * fake hosts to make sure we get many splits in MR jobs. - */ - @Override - public BlockLocation[] getFileBlockLocations(FileStatus file, - long start, long len) throws IOException { - if (file == null) { - return null; - } - - if ((start < 0) || (len < 0)) { - throw new IllegalArgumentException("Invalid start or len parameter"); - } - - if (file.getLen() < start) { - return new BlockLocation[0]; - } - final String blobLocationHost = getConf().get( - AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, - AZURE_BLOCK_LOCATION_HOST_DEFAULT); - final String[] name = { blobLocationHost }; - final String[] host = { blobLocationHost }; - long blockSize = file.getBlockSize(); - if (blockSize <= 0) { - throw new IllegalArgumentException( - "The block size for the given file is not a positive number: " - + blockSize); - } - int numberOfLocations = (int) (len / blockSize) - + ((len % blockSize == 0) ? 0 : 1); - BlockLocation[] locations = new BlockLocation[numberOfLocations]; - for (int i = 0; i < locations.length; i++) { - long currentOffset = start + (i * blockSize); - long currentLength = Math.min(blockSize, start + len - currentOffset); - locations[i] = new BlockLocation(name, host, currentOffset, currentLength); - } - return locations; - } - /** * Set the working directory to the given directory. */ diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java deleted file mode 100644 index b280cac71372d..0000000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.OutputStream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.Test; - -/** - * Test block location logic. - */ -public class TestNativeAzureFileSystemBlockLocations - extends AbstractWasbTestWithTimeout { - @Test - public void testNumberOfBlocks() throws Exception { - Configuration conf = new Configuration(); - conf.set(NativeAzureFileSystem.AZURE_BLOCK_SIZE_PROPERTY_NAME, "500"); - AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount - .createMock(conf); - FileSystem fs = testAccount.getFileSystem(); - Path testFile = createTestFile(fs, 1200); - FileStatus stat = fs.getFileStatus(testFile); - assertEquals(500, stat.getBlockSize()); - testAccount.cleanup(); - } - - @Test - public void testBlockLocationsTypical() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(210, 50, 0, 210); - assertEquals(5, locations.length); - assertEquals("localhost", locations[0].getHosts()[0]); - assertEquals(50, locations[0].getLength()); - assertEquals(10, locations[4].getLength()); - assertEquals(100, locations[2].getOffset()); - } - - @Test - public void testBlockLocationsEmptyFile() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(0, 50, 0, 0); - assertEquals(0, locations.length); - } - - @Test - public void testBlockLocationsSmallFile() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(1, 50, 0, 1); - assertEquals(1, locations.length); - assertEquals(1, locations[0].getLength()); - } - - @Test - public void testBlockLocationsExactBlockSizeMultiple() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(200, 50, 0, 200); - assertEquals(4, locations.length); - assertEquals(150, locations[3].getOffset()); - assertEquals(50, locations[3].getLength()); - } - - @Test - public void testBlockLocationsSubsetOfFile() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(205, 10, 15, 35); - assertEquals(4, locations.length); - assertEquals(10, locations[0].getLength()); - assertEquals(15, locations[0].getOffset()); - assertEquals(5, locations[3].getLength()); - assertEquals(45, locations[3].getOffset()); - } - - @Test - public void testBlockLocationsOutOfRangeSubsetOfFile() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(205, 10, 300, 10); - assertEquals(0, locations.length); - } - - @Test - public void testBlockLocationsEmptySubsetOfFile() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(205, 10, 0, 0); - assertEquals(0, locations.length); - } - - @Test - public void testBlockLocationsDifferentLocationHost() throws Exception { - BlockLocation[] locations = getBlockLocationsOutput(100, 10, 0, 100, - "myblobhost"); - assertEquals(10, locations.length); - assertEquals("myblobhost", locations[0].getHosts()[0]); - } - - private static BlockLocation[] getBlockLocationsOutput(int fileSize, - int blockSize, long start, long len) throws Exception { - return getBlockLocationsOutput(fileSize, blockSize, start, len, null); - } - - private static BlockLocation[] getBlockLocationsOutput(int fileSize, - int blockSize, long start, long len, String blockLocationHost) - throws Exception { - Configuration conf = new Configuration(); - conf.set(NativeAzureFileSystem.AZURE_BLOCK_SIZE_PROPERTY_NAME, "" - + blockSize); - if (blockLocationHost != null) { - conf.set(NativeAzureFileSystem.AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, - blockLocationHost); - } - AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount - .createMock(conf); - FileSystem fs = testAccount.getFileSystem(); - Path testFile = createTestFile(fs, fileSize); - FileStatus stat = fs.getFileStatus(testFile); - BlockLocation[] locations = fs.getFileBlockLocations(stat, start, len); - testAccount.cleanup(); - return locations; - } - - private static Path createTestFile(FileSystem fs, int size) throws Exception { - Path testFile = new Path("/testFile"); - OutputStream outputStream = fs.create(testFile); - outputStream.write(new byte[size]); - outputStream.close(); - return testFile; - } -} From d54e19a913ca53a33970a351d6a3e59bbeac13b4 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 28 Mar 2018 11:37:34 -0700 Subject: [PATCH 034/512] HDFS-13314. NameNode should optionally exit if it detects FsImage corruption. Contributed by Arpit Agarwal. --- .../hadoop/hdfs/server/namenode/FSImage.java | 29 ++++++++-- .../namenode/FSImageFormatProtobuf.java | 31 ++++++++--- .../snapshot/FSImageFormatPBSnapshot.java | 55 ++++++++++++++++++- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index e7581084d1421..dd7df5ad6b4d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -68,6 +69,7 @@ import org.apache.hadoop.hdfs.util.Canceler; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Time; import com.google.common.base.Joiner; @@ -86,6 +88,10 @@ public class FSImage implements Closeable { protected FSEditLog editLog = null; private boolean isUpgradeFinalized = false; + // If true, then image corruption was detected. The NameNode process will + // exit immediately after saving the image. + private AtomicBoolean exitAfterSave = new AtomicBoolean(false); + protected NNStorage storage; /** @@ -954,8 +960,14 @@ void saveFSImage(SaveNamespaceContext context, StorageDirectory sd, FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); - saver.save(newFile, compression); - + long numErrors = saver.save(newFile, compression); + if (numErrors > 0) { + // The image is likely corrupted. + LOG.error("Detected " + numErrors + " errors while saving FsImage " + + dstFile); + exitAfterSave.set(true); + } + MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest()); storage.setMostRecentCheckpointInfo(txid, Time.now()); } @@ -1117,6 +1129,12 @@ public synchronized void saveNamespace(FSNamesystem source, NameNodeFile nnf, } //Update NameDirSize Metric getStorage().updateNameDirSize(); + + if (exitAfterSave.get()) { + LOG.fatal("NameNode process will exit now... The saved FsImage " + + nnf + " is potentially corrupted."); + ExitUtil.terminate(-1); + } } /** @@ -1184,8 +1202,11 @@ private synchronized void saveFSImageInAllDirs(FSNamesystem source, // Since we now have a new checkpoint, we can clean up some // old edit logs and checkpoints. - purgeOldStorage(nnf); - archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW); + // Do not purge anything if we just wrote a corrupted FsImage. + if (!exitAfterSave.get()) { + purgeOldStorage(nnf); + archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW); + } } finally { // Notify any threads waiting on the checkpoint to be canceled // that it is complete. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index cd5a5526cc0cc..4ac20adc4266a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -444,15 +444,22 @@ private void flushSectionOutputStream() throws IOException { sectionOutputStream.flush(); } - void save(File file, FSImageCompression compression) throws IOException { + /** + * @return number of non-fatal errors detected while writing the image. + * @throws IOException on fatal error. + */ + long save(File file, FSImageCompression compression) throws IOException { FileOutputStream fout = new FileOutputStream(file); fileChannel = fout.getChannel(); try { LOG.info("Saving image file {} using {}", file, compression); long startTime = monotonicNow(); - saveInternal(fout, compression, file.getAbsolutePath()); - LOG.info("Image file {} of size {} bytes saved in {} seconds.", file, - file.length(), (monotonicNow() - startTime) / 1000); + long numErrors = saveInternal( + fout, compression, file.getAbsolutePath()); + LOG.info("Image file {} of size {} bytes saved in {} seconds {}.", file, + file.length(), (monotonicNow() - startTime) / 1000, + (numErrors > 0 ? (" with" + numErrors + " errors") : "")); + return numErrors; } finally { fout.close(); } @@ -476,7 +483,11 @@ private void saveInodes(FileSummary.Builder summary) throws IOException { saver.serializeFilesUCSection(sectionOutputStream); } - private void saveSnapshots(FileSummary.Builder summary) throws IOException { + /** + * @return number of non-fatal errors detected while saving the image. + * @throws IOException on fatal error. + */ + private long saveSnapshots(FileSummary.Builder summary) throws IOException { FSImageFormatPBSnapshot.Saver snapshotSaver = new FSImageFormatPBSnapshot.Saver( this, summary, context, context.getSourceNamesystem()); @@ -487,9 +498,14 @@ private void saveSnapshots(FileSummary.Builder summary) throws IOException { snapshotSaver.serializeSnapshotDiffSection(sectionOutputStream); } snapshotSaver.serializeINodeReferenceSection(sectionOutputStream); + return snapshotSaver.getNumImageErrors(); } - private void saveInternal(FileOutputStream fout, + /** + * @return number of non-fatal errors detected while writing the FsImage. + * @throws IOException on fatal error. + */ + private long saveInternal(FileOutputStream fout, FSImageCompression compression, String filePath) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); MessageDigest digester = MD5Hash.getDigester(); @@ -528,7 +544,7 @@ private void saveInternal(FileOutputStream fout, step = new Step(StepType.INODES, filePath); prog.beginStep(Phase.SAVING_CHECKPOINT, step); saveInodes(b); - saveSnapshots(b); + long numErrors = saveSnapshots(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); step = new Step(StepType.DELEGATION_TOKENS, filePath); @@ -551,6 +567,7 @@ private void saveInternal(FileOutputStream fout, saveFileSummary(underlyingOutputStream, summary); underlyingOutputStream.close(); savedDigest = new MD5Hash(digester.digest()); + return numErrors; } private void saveSecretManagerSection(FileSummary.Builder summary) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java index 4a5ceadecebe3..2157554cd62ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java @@ -1,4 +1,4 @@ -/** + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,6 +27,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat; import org.apache.hadoop.hdfs.server.namenode.AclFeature; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext; @@ -415,6 +417,7 @@ public final static class Saver { private final FileSummary.Builder headers; private final FSImageFormatProtobuf.Saver parent; private final SaveNamespaceContext context; + private long numImageErrors; public Saver(FSImageFormatProtobuf.Saver parent, FileSummary.Builder headers, SaveNamespaceContext context, @@ -423,6 +426,7 @@ public Saver(FSImageFormatProtobuf.Saver parent, this.headers = headers; this.context = context; this.fsn = fsn; + this.numImageErrors = 0; } /** @@ -471,15 +475,17 @@ public void serializeINodeReferenceSection(OutputStream out) throws IOException { final List refList = parent.getSaverContext() .getRefList(); + long i = 0; for (INodeReference ref : refList) { - INodeReferenceSection.INodeReference.Builder rb = buildINodeReference(ref); + INodeReferenceSection.INodeReference.Builder rb = + buildINodeReference(ref, i++); rb.build().writeDelimitedTo(out); } parent.commitSection(headers, SectionName.INODE_REFERENCE); } private INodeReferenceSection.INodeReference.Builder buildINodeReference( - INodeReference ref) throws IOException { + final INodeReference ref, final long refIndex) throws IOException { INodeReferenceSection.INodeReference.Builder rb = INodeReferenceSection.INodeReference.newBuilder(). setReferredId(ref.getId()); @@ -489,6 +495,16 @@ private INodeReferenceSection.INodeReference.Builder buildINodeReference( } else if (ref instanceof DstReference) { rb.setDstSnapshotId(ref.getDstSnapshotId()); } + + if (fsn.getFSDirectory().getInode(ref.getId()) == null) { + FSImage.LOG.error( + "FSImageFormatPBSnapshot: Missing referred INodeId " + + ref.getId() + " for INodeReference index " + refIndex + + "; path=" + ref.getFullPathName() + + "; parent=" + (ref.getParent() == null ? "null" : + ref.getParent().getFullPathName())); + ++numImageErrors; + } return rb; } @@ -583,7 +599,23 @@ private void serializeDirDiffList(INodeDirectory dir, List created = diff.getChildrenDiff().getCreatedUnmodifiable(); db.setCreatedListSize(created.size()); List deleted = diff.getChildrenDiff().getDeletedUnmodifiable(); + INode previousNode = null; + boolean misordered = false; for (INode d : deleted) { + // getBytes() may return null below, and that is okay. + final int result = previousNode == null ? -1 : + previousNode.compareTo(d.getLocalNameBytes()); + if (result == 0) { + FSImage.LOG.error( + "Name '" + d.getLocalName() + "' is repeated in the " + + "'deleted' difflist of directory " + + dir.getFullPathName() + ", INodeId=" + dir.getId()); + ++numImageErrors; + } else if (result > 0 && !misordered) { + misordered = true; + ++numImageErrors; + } + previousNode = d; if (d.isReference()) { refList.add(d.asReference()); db.addDeletedINodeRef(refList.size() - 1); @@ -591,11 +623,28 @@ private void serializeDirDiffList(INodeDirectory dir, db.addDeletedINode(d.getId()); } } + if (misordered) { + FSImage.LOG.error( + "Misordered entries in the 'deleted' difflist of directory " + + dir.getFullPathName() + ", INodeId=" + dir.getId() + + ". The full list is " + + Arrays.toString(deleted.toArray())); + } db.build().writeDelimitedTo(out); saveCreatedList(created, out); } } } + + + /** + * Number of non-fatal errors detected while writing the + * SnapshotDiff and INodeReference sections. + * @return the number of non-fatal errors detected. + */ + public long getNumImageErrors() { + return numImageErrors; + } } private FSImageFormatPBSnapshot(){} From 570d7f30720f1e7ac2f7fc7ec58952bed6c9555a Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Wed, 28 Mar 2018 20:17:37 -0400 Subject: [PATCH 035/512] YARN-8069. Clean up example hostnames for RegistryDNS. Contributed by Billie Rinaldi (cherry picked from commit 3d185d62fc392dd5313f8e36b9674b9200638463) --- .../registry/server/dns/TestRegistryDNS.java | 66 +++++++++---------- .../test/resources/0.17.172.in-addr.arpa.zone | 24 +++---- .../markdown/yarn-service/Configurations.md | 4 +- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java index 9c369d0e2a54d..bce73ad4de6a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java @@ -156,7 +156,7 @@ public void initialize() throws Exception { protected Configuration createConfiguration() { Configuration conf = new Configuration(); - conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test"); + conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test"); conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0"); conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS); return conf; @@ -179,39 +179,39 @@ public void testAppRegistration() throws Exception { "/registry/users/root/services/org-apache-slider/test1/", record); // start assessing whether correct records are available - Record[] recs = assertDNSQuery("test1.root.hwx.test."); + Record[] recs = assertDNSQuery("test1.root.dev.test."); assertEquals("wrong result", "192.168.1.5", ((ARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("management-api.test1.root.hwx.test.", 2); - assertEquals("wrong target name", "test1.root.hwx.test.", + recs = assertDNSQuery("management-api.test1.root.dev.test.", 2); + assertEquals("wrong target name", "test1.root.dev.test.", ((CNAMERecord) recs[0]).getTarget().toString()); assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord); - recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", + recs = assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", Type.SRV, 1); assertTrue("not an SRV record", recs[0] instanceof SRVRecord); assertEquals("wrong port", 1026, ((SRVRecord) recs[0]).getPort()); - recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", 2); - assertEquals("wrong target name", "test1.root.hwx.test.", + recs = assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", 2); + assertEquals("wrong target name", "test1.root.dev.test.", ((CNAMERecord) recs[0]).getTarget().toString()); assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord); - recs = assertDNSQuery("http-api.test1.root.hwx.test.", 2); - assertEquals("wrong target name", "test1.root.hwx.test.", + recs = assertDNSQuery("http-api.test1.root.dev.test.", 2); + assertEquals("wrong target name", "test1.root.dev.test.", ((CNAMERecord) recs[0]).getTarget().toString()); assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord); - recs = assertDNSQuery("http-api.test1.root.hwx.test.", Type.SRV, + recs = assertDNSQuery("http-api.test1.root.dev.test.", Type.SRV, 1); assertTrue("not an SRV record", recs[0] instanceof SRVRecord); assertEquals("wrong port", 1027, ((SRVRecord) recs[0]).getPort()); - assertDNSQuery("test1.root.hwx.test.", Type.TXT, 3); - assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", Type.TXT, 1); - assertDNSQuery("http-api.test1.root.hwx.test.", Type.TXT, 1); - assertDNSQuery("management-api.test1.root.hwx.test.", Type.TXT, 1); + assertDNSQuery("test1.root.dev.test.", Type.TXT, 3); + assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", Type.TXT, 1); + assertDNSQuery("http-api.test1.root.dev.test.", Type.TXT, 1); + assertDNSQuery("management-api.test1.root.dev.test.", Type.TXT, 1); } @Test @@ -225,11 +225,11 @@ public void testContainerRegistration() throws Exception { // start assessing whether correct records are available Record[] recs = - assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.hwx.test."); + assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.dev.test."); assertEquals("wrong result", "172.17.0.19", ((ARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1); + recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1); assertTrue("not an ARecord", recs[0] instanceof ARecord); } @@ -243,7 +243,7 @@ public void testContainerRegistrationPersistanceAbsent() throws Exception { record); Name name = - Name.fromString("ctr-e50-1451931954322-0016-01-000002.hwx.test."); + Name.fromString("ctr-e50-1451931954322-0016-01-000002.dev.test."); Record question = Record.newRecord(name, Type.A, DClass.IN); Message query = Message.newQuery(question); byte[] responseBytes = registryDNS.generateReply(query, null); @@ -263,12 +263,12 @@ public void testRecordTTL() throws Exception { // start assessing whether correct records are available Record[] recs = assertDNSQuery( - "ctr-e50-1451931954322-0016-01-000002.hwx.test."); + "ctr-e50-1451931954322-0016-01-000002.dev.test."); assertEquals("wrong result", "172.17.0.19", ((ARecord) recs[0]).getAddress().getHostAddress()); assertEquals("wrong ttl", 30L, recs[0].getTTL()); - recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1); + recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1); assertTrue("not an ARecord", recs[0] instanceof ARecord); assertEquals("wrong ttl", 30L, recs[0].getTTL()); @@ -286,7 +286,7 @@ public void testReverseLookup() throws Exception { // start assessing whether correct records are available Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1); assertEquals("wrong result", - "comp-name.test1.root.hwx.test.", + "comp-name.test1.root.dev.test.", ((PTRRecord) recs[0]).getTarget().toString()); } @@ -294,7 +294,7 @@ public void testReverseLookup() throws Exception { public void testReverseLookupInLargeNetwork() throws Exception { setRegistryDNS(new RegistryDNS("TestRegistry")); Configuration conf = createConfiguration(); - conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test"); + conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test"); conf.set(KEY_DNS_ZONE_SUBNET, "172.17.0.0"); conf.set(KEY_DNS_ZONE_MASK, "255.255.224.0"); conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS); @@ -312,7 +312,7 @@ public void testReverseLookupInLargeNetwork() throws Exception { // start assessing whether correct records are available Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1); assertEquals("wrong result", - "comp-name.test1.root.hwx.test.", + "comp-name.test1.root.dev.test.", ((PTRRecord) recs[0]).getTarget().toString()); } @@ -348,7 +348,7 @@ public void testNoContainerIP() throws Exception { // start assessing whether correct records are available Name name = - Name.fromString("ctr-e50-1451931954322-0016-01-000002.hwx.test."); + Name.fromString("ctr-e50-1451931954322-0016-01-000002.dev.test."); Record question = Record.newRecord(name, Type.A, DClass.IN); Message query = Message.newQuery(question); @@ -431,7 +431,7 @@ public void testDNSKEYRecord() throws Exception { // KeyFactory keyFactory = KeyFactory.getInstance("RSA"); // PublicKey pubKey = keyFactory.generatePublic(keySpec); DNSKEYRecord dnskeyRecord = - new DNSKEYRecord(Name.fromString("hwxstg.site."), DClass.IN, 0, + new DNSKEYRecord(Name.fromString("dev.test."), DClass.IN, 0, DNSKEYRecord.Flags.ZONE_KEY, DNSKEYRecord.Protocol.DNSSEC, DNSSEC.Algorithm.RSASHA256, @@ -486,11 +486,11 @@ public void testAAAALookup() throws Exception { // start assessing whether correct records are available Record[] recs = assertDNSQuery( - "ctr-e50-1451931954322-0016-01-000002.hwx.test.", Type.AAAA, 1); + "ctr-e50-1451931954322-0016-01-000002.dev.test.", Type.AAAA, 1); assertEquals("wrong result", "172.17.0.19", ((AAAARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("comp-name.test1.root.hwx.test.", Type.AAAA, 1); + recs = assertDNSQuery("comp-name.test1.root.dev.test.", Type.AAAA, 1); assertTrue("not an ARecord", recs[0] instanceof AAAARecord); } @@ -504,7 +504,7 @@ public void testNegativeLookup() throws Exception { record); // start assessing whether correct records are available - Name name = Name.fromString("missing.hwx.test."); + Name name = Name.fromString("missing.dev.test."); Record question = Record.newRecord(name, Type.A, DClass.IN); Message query = Message.newQuery(question); @@ -533,7 +533,7 @@ public void testNegativeLookup() throws Exception { public void testReadMasterFile() throws Exception { setRegistryDNS(new RegistryDNS("TestRegistry")); Configuration conf = new Configuration(); - conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test"); + conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test"); conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0"); conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS); conf.set(RegistryConstants.KEY_DNS_ZONES_DIR, @@ -561,17 +561,17 @@ public void testReadMasterFile() throws Exception { // start assessing whether correct records are available Record[] recs = - assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.hwx.test."); + assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.dev.test."); assertEquals("wrong result", "172.17.0.19", ((ARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1); + recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1); assertTrue("not an ARecord", recs[0] instanceof ARecord); // lookup dyanmic reverse records recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1); assertEquals("wrong result", - "comp-name.test1.root.hwx.test.", + "comp-name.test1.root.dev.test.", ((PTRRecord) recs[0]).getTarget().toString()); // now lookup static reverse records @@ -583,7 +583,7 @@ public void testReadMasterFile() throws Exception { byte[] responseBytes = getRegistryDNS().generateReply(query, null); Message response = new Message(responseBytes); recs = response.getSectionArray(Section.ANSWER); - assertEquals("wrong result", "cn005.hwx.test.", + assertEquals("wrong result", "cn005.dev.test.", ((PTRRecord) recs[0]).getTarget().toString()); } @@ -636,7 +636,7 @@ public void testExampleDotCom() throws Exception { public void testExternalCNAMERecord() throws Exception { setRegistryDNS(new RegistryDNS("TestRegistry")); Configuration conf = new Configuration(); - conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test"); + conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test"); conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0"); conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS); conf.set(RegistryConstants.KEY_DNS_ZONES_DIR, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone index 0165f0de25a9c..08071e2728ebe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone @@ -19,18 +19,18 @@ $ORIGIN . $TTL 1800 ; 30 minutes 0.17.172.in-addr.arpa IN SOA ns.hwhq.hortonworks.com. it.hortonworks.com. ( - 2015081000 ; serial - 10800 ; refresh (3 hours) - 900 ; retry (15 minutes) - 1814400 ; expire (3 weeks) - 10800 ; minimum (3 hours) + 2015081000 ; serial + 10800 ; refresh (3 hours) + 900 ; retry (15 minutes) + 1814400 ; expire (3 weeks) + 10800 ; minimum (3 hours) ) - NS ns.hwhq.hortonworks.com. - NS ns2.hwhq.hortonworks.com. + NS ns.hwhq.hortonworks.com. + NS ns2.hwhq.hortonworks.com. $ORIGIN 0.17.172.in-addr.arpa. -5 PTR cn005.hwx.test. -6 PTR cn006.hwx.test. -7 PTR cn007.hwx.test. -8 PTR cn008.hwx.test. -9 PTR cn009.hwx.test. +5 PTR cn005.dev.test. +6 PTR cn006.dev.test. +7 PTR cn007.dev.test. +8 PTR cn008.dev.test. +9 PTR cn009.dev.test. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md index c2e6d260f0f2c..8855075764530 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md @@ -141,12 +141,12 @@ User can use these constants in their configurations to be dynamically substitut ``` Here, `COMPONENT_INSTANCE_NAME` and `SERVICE_NAME` are the constants to be substituted by the system. -Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, user name is `devuser` and domain name is `hwxdev.site`. +Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, user name is `devuser` and domain name is `dev.test`. Then, the config will be substituted by the service AM and written in the config file `/etc/hadoop/hbase-site.xml` inside the container as below: ``` hbase.regionserver.hostname - regionserver-0.hbase.devuser.hwxdev.site + regionserver-0.hbase.devuser.dev.test ``` where `regionserver-0` is the actual component instance name assigned by the system for this container. From 04161bad789fc4d970b9f537803806d4ba02a791 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 29 Mar 2018 15:55:39 +0530 Subject: [PATCH 036/512] YARN-8076. Support to specify application tags in distributed shell. Contributed by Weiwei Yang. (cherry picked from commit 431076f63751f855ab6036ff85825a8552257b93) --- .../applications/distributedshell/Client.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index d6a753a44ffb6..61879d0374cb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -230,6 +230,9 @@ public class Client { // Docker client configuration private String dockerClientConfig = null; + // Application tags + private Set applicationTags = new HashSet<>(); + // Command line options private Options opts; @@ -384,6 +387,7 @@ public Client(Configuration conf) throws Exception { "Placement specification. Please note, if this option is specified," + " The \"num_containers\" option will be ignored. All requested" + " containers will be of type GUARANTEED" ); + opts.addOption("application_tags", true, "Application tags."); } /** @@ -604,6 +608,14 @@ public boolean init(String[] args) throws ParseException { if (cliParser.hasOption("docker_client_config")) { dockerClientConfig = cliParser.getOptionValue("docker_client_config"); } + + if (cliParser.hasOption("application_tags")) { + String applicationTagsStr = cliParser.getOptionValue("application_tags"); + String[] appTags = applicationTagsStr.split(","); + for (String appTag : appTags) { + this.applicationTags.add(appTag.trim()); + } + } return true; } @@ -729,6 +741,9 @@ public boolean run() throws IOException, YarnException { } Set tags = new HashSet(); + if (applicationTags != null) { + tags.addAll(applicationTags); + } if (flowName != null) { tags.add(TimelineUtils.generateFlowNameTag(flowName)); } From 73cd67d7e3b1e3c8449fbabf4c5ab7cb20d8a930 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 29 Mar 2018 21:41:16 +0530 Subject: [PATCH 037/512] YARN-8085. ResourceProfilesManager should be set in RMActiveServiceContext. Contributed by Tao Yang. (cherry picked from commit 7a59d60e0c50e8264e07f6b1dadaeedba676c8c2) --- .../RMActiveServiceContext.java | 11 ++++++ .../server/resourcemanager/RMContextImpl.java | 6 +-- .../resourcemanager/ResourceManager.java | 13 +++---- .../yarn/server/resourcemanager/TestRMHA.java | 39 +++++++++++++++++++ 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 06a1d00d95bbd..66065e33baea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; @@ -111,6 +112,7 @@ public class RMActiveServiceContext { private QueueLimitCalculator queueLimitCalculator; private AllocationTagsManager allocationTagsManager; private PlacementConstraintManager placementConstraintManager; + private ResourceProfilesManager resourceProfilesManager; public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); @@ -513,4 +515,13 @@ public void setContainerQueueLimitCalculator( QueueLimitCalculator limitCalculator) { this.queueLimitCalculator = limitCalculator; } + + public ResourceProfilesManager getResourceProfilesManager() { + return resourceProfilesManager; + } + + public void setResourceProfilesManager( + ResourceProfilesManager resourceProfilesManager) { + this.resourceProfilesManager = resourceProfilesManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 0b6be722ac243..84e0f6f6b58aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -93,8 +93,6 @@ public class RMContextImpl implements RMContext { */ private RMActiveServiceContext activeServiceContext; - private ResourceProfilesManager resourceProfilesManager; - private String proxyHostAndPort = null; /** @@ -591,7 +589,7 @@ public RMAppLifetimeMonitor getRMAppLifetimeMonitor() { @Override public ResourceProfilesManager getResourceProfilesManager() { - return this.resourceProfilesManager; + return this.activeServiceContext.getResourceProfilesManager(); } String getProxyHostAndPort(Configuration conf) { @@ -619,7 +617,7 @@ public String getAppProxyUrl(Configuration conf, ApplicationId applicationId) @Override public void setResourceProfilesManager(ResourceProfilesManager mgr) { - this.resourceProfilesManager = mgr; + this.activeServiceContext.setResourceProfilesManager(mgr); } // Note: Read java doc before adding any services over here. } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 38da7f5e5c324..733da5bd7181c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -245,13 +245,6 @@ protected void serviceInit(Configuration conf) throws Exception { this.rmContext = new RMContextImpl(); rmContext.setResourceManager(this); - - // add resource profiles here because it's used by AbstractYarnScheduler - ResourceProfilesManager resourceProfilesManager = - createResourceProfileManager(); - resourceProfilesManager.init(conf); - rmContext.setResourceProfilesManager(resourceProfilesManager); - this.configurationProvider = ConfigurationProviderFactory.getConfigurationProvider(conf); this.configurationProvider.init(this.conf); @@ -655,6 +648,12 @@ protected void serviceInit(Configuration configuration) throws Exception { addService(placementConstraintManager); rmContext.setPlacementConstraintManager(placementConstraintManager); + // add resource profiles here because it's used by AbstractYarnScheduler + ResourceProfilesManager resourceProfilesManager = + createResourceProfileManager(); + resourceProfilesManager.init(conf); + rmContext.setResourceProfilesManager(resourceProfilesManager); + RMDelegatedNodeLabelsUpdater delegatedNodeLabelsUpdater = createRMDelegatedNodeLabelsUpdater(); if (delegatedNodeLabelsUpdater != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 20e9ff4e650d2..385e8dbedd3b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -658,6 +658,45 @@ protected Dispatcher createDispatcher() { assertEquals(HAServiceState.STANDBY, rm.getRMContext().getHAServiceState()); } + @Test + public void testResourceProfilesManagerAfterRMWentStandbyThenBackToActive() + throws Exception { + configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); + configuration.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); + Configuration conf = new YarnConfiguration(configuration); + conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); + + // 1. start RM + rm = new MockRM(conf); + rm.init(conf); + rm.start(); + + StateChangeRequestInfo requestInfo = new StateChangeRequestInfo( + HAServiceProtocol.RequestSource.REQUEST_BY_USER); + checkMonitorHealth(); + checkStandbyRMFunctionality(); + + // 2. Transition to active + rm.adminService.transitionToActive(requestInfo); + checkMonitorHealth(); + checkActiveRMFunctionality(); + + // 3. Transition to standby + rm.adminService.transitionToStandby(requestInfo); + checkMonitorHealth(); + checkStandbyRMFunctionality(); + + // 4. Transition to active + rm.adminService.transitionToActive(requestInfo); + checkMonitorHealth(); + checkActiveRMFunctionality(); + + // 5. Check ResourceProfilesManager + Assert.assertNotNull( + "ResourceProfilesManager should not be null!", + rm.getRMContext().getResourceProfilesManager()); + } + public void innerTestHAWithRMHostName(boolean includeBindHost) { //this is run two times, with and without a bind host configured if (includeBindHost) { From b038e2144085bc237b6d1a8fa96932e4415d3980 Mon Sep 17 00:00:00 2001 From: Sean Mackrory Date: Tue, 27 Mar 2018 15:51:44 -0600 Subject: [PATCH 038/512] HADOOP-15342. Updating ADLS connector to use the current SDK version (2.2.7). Contributed by Atul Sikaria. (cherry picked from commit 9d7a9031a5978efc8d97566e35ebaace20db2353) --- hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index b857abca54891..05e3badafd0c2 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -33,7 +33,7 @@ 0.9.1 UTF-8 true - 2.2.5 + 2.2.7     From b64bf561315d1fae10b986058e201f6a32cbd53f Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Thu, 29 Mar 2018 15:03:31 -0700 Subject: [PATCH 039/512] YARN-8086. ManagedParentQueue with no leaf queues cause JS error in new UI. (Suma Shivaprasad via wangda) Change-Id: I6d82af91adec02698afddde6883b1fe1924e6041 (cherry picked from commit e7e20190dface4117abe70dcfd96d19b0e4f5339) --- .../main/webapp/app/serializers/yarn-queue/capacity-queue.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js index e838255ba1d23..57714e3d46229 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js @@ -24,7 +24,7 @@ export default DS.JSONAPISerializer.extend({ normalizeSingleResponse(store, primaryModelClass, payload, id, requestType) { var children = []; - if (payload.queues) { + if (payload.queues && payload.queues.queue) { payload.queues.queue.forEach(function(queue) { children.push(queue.queueName); }); @@ -122,7 +122,7 @@ export default DS.JSONAPISerializer.extend({ data.push(result.queue); includedData = includedData.concat(result.includedData); - if (payload.queues) { + if (payload.queues && payload.queues.queue) { for (var i = 0; i < payload.queues.queue.length; i++) { var queue = payload.queues.queue[i]; queue.myParent = payload.queueName; From 2960592c6f9f2226b5a60f4ce4717638ccecb823 Mon Sep 17 00:00:00 2001 From: Xiao Chen Date: Thu, 29 Mar 2018 15:36:31 -0700 Subject: [PATCH 040/512] HDFS-13087. Snapshotted encryption zone information should be immutable. Contributed by LiXin Ge. (cherry picked from commit 2c6cfad5a31ca4d9126ecd2b3c43cca8543aacb4) --- .../namenode/EncryptionZoneManager.java | 74 +++++++++++++++++-- .../namenode/FSDirEncryptionZoneOp.java | 15 ++-- .../hdfs/server/namenode/FSDirXAttrOp.java | 8 +- .../hdfs/server/namenode/XAttrStorage.java | 9 +-- .../hadoop/hdfs/TestEncryptionZones.java | 15 +++- 5 files changed, 97 insertions(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index 176ae1dc00ee1..b1bca98406611 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -33,6 +33,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoProtocolVersion; @@ -50,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.security.AccessControlException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +111,34 @@ CipherSuite getSuite() { String getKeyName() { return keyName; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof EncryptionZoneInt)) { + return false; + } + + EncryptionZoneInt b = (EncryptionZoneInt)o; + return new EqualsBuilder() + .append(inodeId, b.getINodeId()) + .append(suite, b.getSuite()) + .append(version, b.getVersion()) + .append(keyName, b.getKeyName()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(). + append(inodeId). + append(suite). + append(version). + append(keyName). + toHashCode(); + } } private TreeMap encryptionZones = null; @@ -315,8 +347,8 @@ void removeEncryptionZone(Long inodeId) { *

    * Called while holding the FSDirectory lock. */ - boolean isInAnEZ(INodesInPath iip) - throws UnresolvedLinkException, SnapshotAccessControlException { + boolean isInAnEZ(INodesInPath iip) throws UnresolvedLinkException, + SnapshotAccessControlException, IOException { assert dir.hasReadLock(); return (getEncryptionZoneForPath(iip) != null); } @@ -341,7 +373,7 @@ String getFullPathName(Long nodeId) { *

    * Called while holding the FSDirectory lock. */ - String getKeyName(final INodesInPath iip) { + String getKeyName(final INodesInPath iip) throws IOException { assert dir.hasReadLock(); EncryptionZoneInt ezi = getEncryptionZoneForPath(iip); if (ezi == null) { @@ -356,19 +388,43 @@ String getKeyName(final INodesInPath iip) { *

    * Called while holding the FSDirectory lock. */ - private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) { + private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) + throws IOException{ assert dir.hasReadLock(); Preconditions.checkNotNull(iip); if (!hasCreatedEncryptionZone()) { return null; } + + int snapshotID = iip.getPathSnapshotId(); for (int i = iip.length() - 1; i >= 0; i--) { final INode inode = iip.getINode(i); - if (inode != null) { + if (inode == null || !inode.isDirectory()) { + //not found or not a directory, encryption zone is supported on + //directory only. + continue; + } + if (snapshotID == Snapshot.CURRENT_STATE_ID) { final EncryptionZoneInt ezi = encryptionZones.get(inode.getId()); if (ezi != null) { return ezi; } + } else { + XAttr xAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName( + inode, snapshotID, CRYPTO_XATTR_ENCRYPTION_ZONE); + if (xAttr != null) { + try { + final HdfsProtos.ZoneEncryptionInfoProto ezProto = + HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xAttr.getValue()); + return new EncryptionZoneInt( + inode.getId(), PBHelperClient.convert(ezProto.getSuite()), + PBHelperClient.convert(ezProto.getCryptoProtocolVersion()), + ezProto.getKeyName()); + } catch (InvalidProtocolBufferException e) { + throw new IOException("Could not parse encryption zone for inode " + + iip.getPath(), e); + } + } } } return null; @@ -381,7 +437,8 @@ private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) { *

    * Called while holding the FSDirectory lock. */ - private EncryptionZoneInt getParentEncryptionZoneForPath(INodesInPath iip) { + private EncryptionZoneInt getParentEncryptionZoneForPath(INodesInPath iip) + throws IOException { assert dir.hasReadLock(); Preconditions.checkNotNull(iip); INodesInPath parentIIP = iip.getParentINodesInPath(); @@ -395,7 +452,8 @@ private EncryptionZoneInt getParentEncryptionZoneForPath(INodesInPath iip) { * @param iip The INodesInPath of the path to check * @return the EncryptionZone representing the ez for the path. */ - EncryptionZone getEZINodeForPath(INodesInPath iip) { + EncryptionZone getEZINodeForPath(INodesInPath iip) + throws IOException { final EncryptionZoneInt ezi = getEncryptionZoneForPath(iip); if (ezi == null) { return null; @@ -437,7 +495,7 @@ void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP) } if (srcInEZ) { - if (srcParentEZI != dstParentEZI) { + if (!srcParentEZI.equals(dstParentEZI)) { final String srcEZPath = getFullPathName(srcParentEZI.getINodeId()); final String dstEZPath = getFullPathName(dstParentEZI.getINodeId()); final StringBuilder sb = new StringBuilder(srcIIP.getPath()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java index bf5652d6fe025..3d78172f9234e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java @@ -205,7 +205,7 @@ static Map.Entry getEZForPath( } static EncryptionZone getEZForPath(final FSDirectory fsd, - final INodesInPath iip) { + final INodesInPath iip) throws IOException { fsd.readLock(); try { return fsd.ezManager.getEZINodeForPath(iip); @@ -360,8 +360,9 @@ static XAttr generateNewXAttrForReencryptionFinish(final INodesInPath iip, private static ZoneEncryptionInfoProto getZoneEncryptionInfoProto( final INodesInPath iip) throws IOException { - final XAttr fileXAttr = FSDirXAttrOp - .unprotectedGetXAttrByPrefixedName(iip, CRYPTO_XATTR_ENCRYPTION_ZONE); + final XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName( + iip.getLastINode(), iip.getPathSnapshotId(), + CRYPTO_XATTR_ENCRYPTION_ZONE); if (fileXAttr == null) { throw new IOException( "Could not find reencryption XAttr for file " + iip.getPath()); @@ -457,7 +458,8 @@ static FileEncryptionInfo getFileEncryptionInfo(final FSDirectory fsd, } XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName( - iip, CRYPTO_XATTR_FILE_ENCRYPTION_INFO); + iip.getLastINode(), iip.getPathSnapshotId(), + CRYPTO_XATTR_FILE_ENCRYPTION_INFO); if (fileXAttr == null) { NameNode.LOG.warn("Could not find encryption XAttr for file " + iip.getPath() + " in encryption zone " + encryptionZone.getPath()); @@ -494,7 +496,7 @@ static FileEncryptionInfo getFileEncryptionInfo(final FSDirectory fsd, */ static FileEncryptionInfo getFileEncryptionInfo(FSDirectory dir, INodesInPath iip, EncryptionKeyInfo ezInfo) - throws RetryStartFileException { + throws RetryStartFileException, IOException { FileEncryptionInfo feInfo = null; final EncryptionZone zone = getEZForPath(dir, iip); if (zone != null) { @@ -517,7 +519,8 @@ static FileEncryptionInfo getFileEncryptionInfo(FSDirectory dir, } static boolean isInAnEZ(final FSDirectory fsd, final INodesInPath iip) - throws UnresolvedLinkException, SnapshotAccessControlException { + throws UnresolvedLinkException, SnapshotAccessControlException, + IOException { if (!fsd.ezManager.hasCreatedEncryptionZone()) { return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index 24a475fcc0094..9e95f90d8663b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -378,16 +378,18 @@ static XAttr getXAttrByPrefixedName(FSDirectory fsd, INodesInPath iip, String prefixedName) throws IOException { fsd.readLock(); try { - return XAttrStorage.readINodeXAttrByPrefixedName(iip, prefixedName); + return XAttrStorage.readINodeXAttrByPrefixedName(iip.getLastINode(), + iip.getPathSnapshotId(), prefixedName); } finally { fsd.readUnlock(); } } static XAttr unprotectedGetXAttrByPrefixedName( - INodesInPath iip, String prefixedName) + INode inode, int snapshotId, String prefixedName) throws IOException { - return XAttrStorage.readINodeXAttrByPrefixedName(iip, prefixedName); + return XAttrStorage.readINodeXAttrByPrefixedName( + inode, snapshotId, prefixedName); } private static void checkXAttrChangeAccess( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java index 8a91e2a723bfc..3b3747b1d8508 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java @@ -47,14 +47,13 @@ public static String getName(int n) { *

    * * @param inode INode to read - * @param snapshotId + * @param snapshotId the snapshotId of the requested path * @param prefixedName xAttr name with prefix * @return the xAttr */ - public static XAttr readINodeXAttrByPrefixedName(INodesInPath iip, - String prefixedName) { - XAttrFeature f = - iip.getLastINode().getXAttrFeature(iip.getPathSnapshotId()); + public static XAttr readINodeXAttrByPrefixedName(INode inode, int snapshotId, + String prefixedName) { + XAttrFeature f = inode.getXAttrFeature(snapshotId); return f == null ? null : f.getXAttr(prefixedName); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 4e69b9d99ad37..8723a56e5aa7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -1410,11 +1410,20 @@ public void testSnapshotsOnEncryptionZones() throws Exception { fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true); final Path snap2 = fs.createSnapshot(zoneParent, "snap2"); final Path snap2Zone = new Path(snap2, zone.getName()); + assertEquals("Got unexpected ez path", zone.toString(), + dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString()); assertNull("Expected null ez path", dfsAdmin.getEncryptionZoneForPath(snap2Zone)); - // Create the encryption zone again + // Create the encryption zone again, and that shouldn't affect old snapshot dfsAdmin.createEncryptionZone(zone, TEST_KEY2, NO_TRASH); + EncryptionZone ezSnap1 = dfsAdmin.getEncryptionZoneForPath(snap1Zone); + assertEquals("Got unexpected ez path", zone.toString(), + ezSnap1.getPath().toString()); + assertEquals("Unexpected ez key", TEST_KEY, ezSnap1.getKeyName()); + assertNull("Expected null ez path", + dfsAdmin.getEncryptionZoneForPath(snap2Zone)); + final Path snap3 = fs.createSnapshot(zoneParent, "snap3"); final Path snap3Zone = new Path(snap3, zone.getName()); // Check that snap3's EZ has the correct settings @@ -1423,10 +1432,12 @@ public void testSnapshotsOnEncryptionZones() throws Exception { ezSnap3.getPath().toString()); assertEquals("Unexpected ez key", TEST_KEY2, ezSnap3.getKeyName()); // Check that older snapshots still have the old EZ settings - EncryptionZone ezSnap1 = dfsAdmin.getEncryptionZoneForPath(snap1Zone); + ezSnap1 = dfsAdmin.getEncryptionZoneForPath(snap1Zone); assertEquals("Got unexpected ez path", zone.toString(), ezSnap1.getPath().toString()); assertEquals("Unexpected ez key", TEST_KEY, ezSnap1.getKeyName()); + assertNull("Expected null ez path", + dfsAdmin.getEncryptionZoneForPath(snap2Zone)); // Check that listEZs only shows the current filesystem state ArrayList listZones = Lists.newArrayList(); From 99b5b9dce1b4a17ce4454c169126ae1a8b2cae28 Mon Sep 17 00:00:00 2001 From: Konstantin V Shvachko Date: Thu, 29 Mar 2018 17:13:18 -0700 Subject: [PATCH 041/512] HADOOP-12862. LDAP Group Mapping over SSL can not specify trust store. Contributed by Wei-Chiu Chuang and Konstantin Shvachko. (cherry picked from commit 2216bde322961c0fe33b5822510880a65d5c45fd) --- .../org/apache/hadoop/conf/Configuration.java | 2 +- .../hadoop/security/LdapGroupsMapping.java | 90 ++++++++++++++++--- .../src/main/resources/core-default.xml | 21 +++++ .../src/site/markdown/GroupsMapping.md | 3 + 4 files changed, 105 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 00b4702de4bbc..65573564fb6db 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -2332,7 +2332,7 @@ private CredentialEntry getCredentialEntry(CredentialProvider provider, * @return password or null if not found * @throws IOException */ - protected char[] getPasswordFromCredentialProviders(String name) + public char[] getPasswordFromCredentialProviders(String name) throws IOException { char[] pass = null; try { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java index babfa3809b6a0..6beaa9e98f807 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java @@ -109,6 +109,27 @@ public class LdapGroupsMapping public static final String LDAP_KEYSTORE_PASSWORD_FILE_KEY = LDAP_KEYSTORE_PASSWORD_KEY + ".file"; public static final String LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT = ""; + + /** + * File path to the location of the SSL truststore to use + */ + public static final String LDAP_TRUSTSTORE_KEY = LDAP_CONFIG_PREFIX + + ".ssl.truststore"; + + /** + * The key of the credential entry containing the password for + * the LDAP SSL truststore + */ + public static final String LDAP_TRUSTSTORE_PASSWORD_KEY = + LDAP_CONFIG_PREFIX +".ssl.truststore.password"; + + /** + * The path to a file containing the password for + * the LDAP SSL truststore + */ + public static final String LDAP_TRUSTSTORE_PASSWORD_FILE_KEY = + LDAP_TRUSTSTORE_PASSWORD_KEY + ".file"; + /* * User to bind to the LDAP server with */ @@ -226,6 +247,8 @@ public class LdapGroupsMapping private boolean useSsl; private String keystore; private String keystorePass; + private String truststore; + private String truststorePass; private String bindUser; private String bindPassword; private String userbaseDN; @@ -526,8 +549,19 @@ DirContext getDirContext() throws NamingException { // Set up SSL security, if necessary if (useSsl) { env.put(Context.SECURITY_PROTOCOL, "ssl"); - System.setProperty("javax.net.ssl.keyStore", keystore); - System.setProperty("javax.net.ssl.keyStorePassword", keystorePass); + if (!keystore.isEmpty()) { + System.setProperty("javax.net.ssl.keyStore", keystore); + } + if (!keystorePass.isEmpty()) { + System.setProperty("javax.net.ssl.keyStorePassword", keystorePass); + } + if (!truststore.isEmpty()) { + System.setProperty("javax.net.ssl.trustStore", truststore); + } + if (!truststorePass.isEmpty()) { + System.setProperty("javax.net.ssl.trustStorePassword", + truststorePass); + } } env.put(Context.SECURITY_PRINCIPAL, bindUser); @@ -572,15 +606,10 @@ public synchronized void setConf(Configuration conf) { if (ldapUrl == null || ldapUrl.isEmpty()) { throw new RuntimeException("LDAP URL is not configured"); } - + useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT); - keystore = conf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT); - - keystorePass = getPassword(conf, LDAP_KEYSTORE_PASSWORD_KEY, - LDAP_KEYSTORE_PASSWORD_DEFAULT); - if (keystorePass.isEmpty()) { - keystorePass = extractPassword(conf.get(LDAP_KEYSTORE_PASSWORD_FILE_KEY, - LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT)); + if (useSsl) { + loadSslConf(conf); } bindUser = conf.get(BIND_USER_KEY, BIND_USER_DEFAULT); @@ -643,6 +672,47 @@ public synchronized void setConf(Configuration conf) { this.conf = conf; } + private void loadSslConf(Configuration sslConf) { + keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT); + keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY, + LDAP_KEYSTORE_PASSWORD_DEFAULT); + if (keystorePass.isEmpty()) { + keystorePass = extractPassword(sslConf.get( + LDAP_KEYSTORE_PASSWORD_FILE_KEY, + LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT)); + } + + truststore = sslConf.get(LDAP_TRUSTSTORE_KEY, ""); + truststorePass = getPasswordFromCredentialProviders( + sslConf, LDAP_TRUSTSTORE_PASSWORD_KEY, ""); + if (truststorePass.isEmpty()) { + truststorePass = extractPassword( + sslConf.get(LDAP_TRUSTSTORE_PASSWORD_FILE_KEY, "")); + } + } + + String getPasswordFromCredentialProviders( + Configuration conf, String alias, String defaultPass) { + String password = defaultPass; + try { + char[] passchars = conf.getPasswordFromCredentialProviders(alias); + if (passchars != null) { + password = new String(passchars); + } + } catch (IOException ioe) { + LOG.warn("Exception while trying to get password for alias {}: {}", + alias, ioe); + } + return password; + } + + /** + * Passwords should not be stored in configuration. Use + * {@link #getPasswordFromCredentialProviders( + * Configuration, String, String)} + * to avoid reading passwords from a configuration file. + */ + @Deprecated String getPassword(Configuration conf, String alias, String defaultPass) { String password = defaultPass; try { diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 49bfa2881c98b..ad24f5624325f 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -316,6 +316,27 @@ + + hadoop.security.group.mapping.ldap.ssl.truststore + + + File path to the SSL truststore that contains the root certificate used to + sign the LDAP server's certificate. Specify this if the LDAP server's + certificate is not signed by a well known certificate authority. + + + + + hadoop.security.group.mapping.ldap.ssl.truststore.password.file + + + The path to a file containing the password of the LDAP SSL truststore. + + IMPORTANT: This file should be readable only by the Unix user running + the daemons. + + + hadoop.security.group.mapping.ldap.bind.user diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md index 806ed54225b8e..c6af930030a62 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md @@ -112,6 +112,9 @@ For some LDAP servers, such as Active Directory, the user object returned in the Therefore, it is possible to infer the user's groups from the first query without sending the second one, and it may reduce group name resolution latency incurred by the second query. If it fails to get group names, it will fall back to the typical two-query scenario and send the second query to get group names. To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memberof` to `memberOf`, and Hadoop will resolve group names using this attribute in the user object. +If the LDAP server's certificate is not signed by a well known certificate authority, specify the path to the truststore in `hadoop.security.group.mapping.ldap.ssl.truststore`. +Similar to keystore, specify the truststore password file in `hadoop.security.group.mapping.ldap.ssl.truststore.password.file`. + Composite Groups Mapping -------- `CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`. From e96c7bf82de1e9fd97df5fb6b763e211ebad5913 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Fri, 30 Mar 2018 20:23:05 +0800 Subject: [PATCH 042/512] HADOOP-14999. AliyunOSS: provide one asynchronous multi-part based uploading mechanism. Contributed by Genmao Yu. (cherry picked from commit 6542d17ea460ec222137c4b275b13daf15d3fca3) --- .../aliyun/oss/AliyunCredentialsProvider.java | 3 +- .../oss/AliyunOSSBlockOutputStream.java | 206 ++++++++++++++++++ .../fs/aliyun/oss/AliyunOSSFileSystem.java | 34 ++- .../aliyun/oss/AliyunOSSFileSystemStore.java | 173 ++++++++------- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 111 ---------- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 115 +++++++--- .../hadoop/fs/aliyun/oss/Constants.java | 22 +- ...va => TestAliyunOSSBlockOutputStream.java} | 32 ++- .../aliyun/oss/TestAliyunOSSInputStream.java | 10 +- .../contract/TestAliyunOSSContractDistCp.java | 2 +- 10 files changed, 457 insertions(+), 251 deletions(-) create mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java delete mode 100644 hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java rename hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/{TestAliyunOSSOutputStream.java => TestAliyunOSSBlockOutputStream.java} (70%) diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java index b46c67aa5e7e0..58c14a943bc2d 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java @@ -35,8 +35,7 @@ public class AliyunCredentialsProvider implements CredentialsProvider { private Credentials credentials = null; - public AliyunCredentialsProvider(Configuration conf) - throws IOException { + public AliyunCredentialsProvider(Configuration conf) throws IOException { String accessKeyId; String accessKeySecret; String securityToken; diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java new file mode 100644 index 0000000000000..12d551bbc285a --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.model.PartETag; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; + +/** + * Asynchronous multi-part based uploading mechanism to support huge file + * which is larger than 5GB. Data will be buffered on local disk, then uploaded + * to OSS in {@link #close()} method. + */ +public class AliyunOSSBlockOutputStream extends OutputStream { + private static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSBlockOutputStream.class); + private AliyunOSSFileSystemStore store; + private Configuration conf; + private boolean closed; + private String key; + private File blockFile; + private List blockFiles = new ArrayList<>(); + private long blockSize; + private int blockId = 0; + private long blockWritten = 0L; + private String uploadId = null; + private final List> partETagsFutures; + private final ListeningExecutorService executorService; + private OutputStream blockStream; + private final byte[] singleByte = new byte[1]; + + public AliyunOSSBlockOutputStream(Configuration conf, + AliyunOSSFileSystemStore store, + String key, + Long blockSize, + ExecutorService executorService) throws IOException { + this.store = store; + this.conf = conf; + this.key = key; + this.blockSize = blockSize; + this.blockFile = newBlockFile(); + this.blockStream = + new BufferedOutputStream(new FileOutputStream(blockFile)); + this.partETagsFutures = new ArrayList<>(2); + this.executorService = MoreExecutors.listeningDecorator(executorService); + } + + private File newBlockFile() throws IOException { + return AliyunOSSUtils.createTmpFileForWrite( + String.format("oss-block-%04d-", blockId), blockSize, conf); + } + + @Override + public synchronized void flush() throws IOException { + blockStream.flush(); + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + + blockStream.flush(); + blockStream.close(); + if (!blockFiles.contains(blockFile)) { + blockFiles.add(blockFile); + } + + try { + if (blockFiles.size() == 1) { + // just upload it directly + store.uploadObject(key, blockFile); + } else { + if (blockWritten > 0) { + ListenableFuture partETagFuture = + executorService.submit(() -> { + PartETag partETag = store.uploadPart(blockFile, key, uploadId, + blockId + 1); + return partETag; + }); + partETagsFutures.add(partETagFuture); + } + // wait for the partial uploads to finish + final List partETags = waitForAllPartUploads(); + if (null == partETags) { + throw new IOException("Failed to multipart upload to oss, abort it."); + } + store.completeMultipartUpload(key, uploadId, partETags); + } + } finally { + for (File tFile: blockFiles) { + if (tFile.exists() && !tFile.delete()) { + LOG.warn("Failed to delete temporary file {}", tFile); + } + } + closed = true; + } + } + + @Override + public void write(int b) throws IOException { + singleByte[0] = (byte)b; + write(singleByte, 0, 1); + } + + @Override + public synchronized void write(byte[] b, int off, int len) + throws IOException { + if (closed) { + throw new IOException("Stream closed."); + } + try { + blockStream.write(b, off, len); + blockWritten += len; + if (blockWritten >= blockSize) { + uploadCurrentPart(); + blockWritten = 0L; + } + } finally { + for (File tFile: blockFiles) { + if (tFile.exists() && !tFile.delete()) { + LOG.warn("Failed to delete temporary file {}", tFile); + } + } + } + } + + private void uploadCurrentPart() throws IOException { + blockFiles.add(blockFile); + blockStream.flush(); + blockStream.close(); + if (blockId == 0) { + uploadId = store.getUploadId(key); + } + ListenableFuture partETagFuture = + executorService.submit(() -> { + PartETag partETag = store.uploadPart(blockFile, key, uploadId, + blockId + 1); + return partETag; + }); + partETagsFutures.add(partETagFuture); + blockFile = newBlockFile(); + blockId++; + blockStream = new BufferedOutputStream(new FileOutputStream(blockFile)); + } + + /** + * Block awaiting all outstanding uploads to complete. + * @return list of results + * @throws IOException IO Problems + */ + private List waitForAllPartUploads() throws IOException { + LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size()); + try { + return Futures.allAsList(partETagsFutures).get(); + } catch (InterruptedException ie) { + LOG.warn("Interrupted partUpload", ie); + Thread.currentThread().interrupt(); + return null; + } catch (ExecutionException ee) { + //there is no way of recovering so abort + //cancel all partUploads + LOG.debug("While waiting for upload completion", ee); + LOG.debug("Cancelling futures"); + for (ListenableFuture future : partETagsFutures) { + future.cancel(true); + } + //abort multipartupload + store.abortMultipartUpload(key, uploadId); + throw new IOException("Multi-part upload with id '" + uploadId + + "' to " + key, ee); + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index b3c63d33535ea..93e31d57e838a 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -56,6 +56,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.intOption; +import static org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.longOption; import static org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.objectRepresentsDirectory; import static org.apache.hadoop.fs.aliyun.oss.Constants.*; @@ -69,6 +71,7 @@ public class AliyunOSSFileSystem extends FileSystem { private URI uri; private String bucket; private Path workingDir; + private int blockOutputActiveBlocks; private AliyunOSSFileSystemStore store; private int maxKeys; private int maxReadAheadPartNumber; @@ -125,8 +128,15 @@ public FSDataOutputStream create(Path path, FsPermission permission, // this means the file is not found } - return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(), - store, key, progress, statistics), (Statistics)(null)); + long uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(getConf(), + MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT); + return new FSDataOutputStream( + new AliyunOSSBlockOutputStream(getConf(), + store, + key, + uploadPartSize, + new SemaphoredDelegatingExecutor(boundedThreadPool, + blockOutputActiveBlocks, true)), (Statistics)(null)); } /** @@ -149,9 +159,8 @@ public FSDataOutputStream createNonRecursive(Path path, throw new FileAlreadyExistsException("Not a directory: " + parent); } } - return create(path, permission, - flags.contains(CreateFlag.OVERWRITE), bufferSize, - replication, blockSize, progress); + return create(path, permission, flags.contains(CreateFlag.OVERWRITE), + bufferSize, replication, blockSize, progress); } @Override @@ -270,7 +279,7 @@ public FileStatus getFileStatus(Path path) throws IOException { } } else if (objectRepresentsDirectory(key, meta.getContentLength())) { return new FileStatus(0, true, 1, 0, meta.getLastModified().getTime(), - qualifiedPath); + qualifiedPath); } else { return new FileStatus(meta.getContentLength(), false, 1, getDefaultBlockSize(path), meta.getLastModified().getTime(), @@ -318,6 +327,10 @@ public void initialize(URI name, Configuration conf) throws IOException { uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority()); workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(uri, null); + long keepAliveTime = longOption(conf, + KEEPALIVE_TIME_KEY, KEEPALIVE_TIME_DEFAULT, 0); + blockOutputActiveBlocks = intOption(conf, + UPLOAD_ACTIVE_BLOCKS_KEY, UPLOAD_ACTIVE_BLOCKS_DEFAULT, 1); store = new AliyunOSSFileSystemStore(); store.initialize(name, conf, statistics); @@ -335,7 +348,8 @@ public void initialize(URI name, Configuration conf) throws IOException { Constants.MULTIPART_DOWNLOAD_AHEAD_PART_MAX_NUM_DEFAULT); this.boundedThreadPool = BlockingThreadPoolExecutorService.newInstance( - threadNum, totalTasks, 60L, TimeUnit.SECONDS, "oss-read-shared"); + threadNum, totalTasks, keepAliveTime, TimeUnit.SECONDS, + "oss-transfer-shared"); maxConcurrentCopyTasksPerDir = AliyunOSSUtils.intPositiveOption(conf, Constants.MAX_CONCURRENT_COPY_TASKS_PER_DIR_KEY, @@ -490,12 +504,12 @@ private RemoteIterator innerList(final Path f, if (status.isFile()) { LOG.debug("{} is a File", qualifiedPath); final BlockLocation[] locations = getFileBlockLocations(status, - 0, status.getLen()); + 0, status.getLen()); return store.singleStatusRemoteIterator(filter.accept(f) ? status : null, - locations); + locations); } else { return store.createLocatedFileStatusIterator(key, maxKeys, this, filter, - acceptor, recursive ? null : "/"); + acceptor, recursive ? null : "/"); } } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java index a7f13c082cd0f..cc050c876ec53 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.hadoop.fs.aliyun.oss; import com.aliyun.oss.ClientConfiguration; @@ -62,8 +63,11 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.net.URI; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.ListIterator; import java.util.NoSuchElementException; @@ -83,7 +87,6 @@ public class AliyunOSSFileSystemStore { private String bucketName; private long uploadPartSize; private long multipartThreshold; - private long partSize; private int maxKeys; private String serverSideEncryptionAlgorithm; @@ -143,28 +146,18 @@ public void initialize(URI uri, Configuration conf, String endPoint = conf.getTrimmed(ENDPOINT_KEY, ""); if (StringUtils.isEmpty(endPoint)) { throw new IllegalArgumentException("Aliyun OSS endpoint should not be " + - "null or empty. Please set proper endpoint with 'fs.oss.endpoint'."); + "null or empty. Please set proper endpoint with 'fs.oss.endpoint'."); } CredentialsProvider provider = AliyunOSSUtils.getCredentialsProvider(conf); ossClient = new OSSClient(endPoint, provider, clientConf); - uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, - MULTIPART_UPLOAD_SIZE_DEFAULT); + uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf, + MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT); multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); - partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, - MULTIPART_UPLOAD_SIZE_DEFAULT); - if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) { - partSize = MIN_MULTIPART_UPLOAD_PART_SIZE; - } serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, ""); - if (uploadPartSize < 5 * 1024 * 1024) { - LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB"); - uploadPartSize = 5 * 1024 * 1024; - } - if (multipartThreshold < 5 * 1024 * 1024) { LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB"); multipartThreshold = 5 * 1024 * 1024; @@ -419,71 +412,6 @@ public void uploadObject(String key, File file) throws IOException { } } - /** - * Upload a file as an OSS object, using multipart upload. - * - * @param key object key. - * @param file local file to upload. - * @throws IOException if failed to upload object. - */ - public void multipartUploadObject(String key, File file) throws IOException { - File object = file.getAbsoluteFile(); - long dataLen = object.length(); - long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize); - int partNum = (int) (dataLen / realPartSize); - if (dataLen % realPartSize != 0) { - partNum += 1; - } - - InitiateMultipartUploadRequest initiateMultipartUploadRequest = - new InitiateMultipartUploadRequest(bucketName, key); - ObjectMetadata meta = new ObjectMetadata(); - if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { - meta.setServerSideEncryption(serverSideEncryptionAlgorithm); - } - initiateMultipartUploadRequest.setObjectMetadata(meta); - InitiateMultipartUploadResult initiateMultipartUploadResult = - ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); - List partETags = new ArrayList(); - String uploadId = initiateMultipartUploadResult.getUploadId(); - - try { - for (int i = 0; i < partNum; i++) { - // TODO: Optimize this, avoid opening the object multiple times - FileInputStream fis = new FileInputStream(object); - try { - long skipBytes = realPartSize * i; - AliyunOSSUtils.skipFully(fis, skipBytes); - long size = (realPartSize < dataLen - skipBytes) ? - realPartSize : dataLen - skipBytes; - UploadPartRequest uploadPartRequest = new UploadPartRequest(); - uploadPartRequest.setBucketName(bucketName); - uploadPartRequest.setKey(key); - uploadPartRequest.setUploadId(uploadId); - uploadPartRequest.setInputStream(fis); - uploadPartRequest.setPartSize(size); - uploadPartRequest.setPartNumber(i + 1); - UploadPartResult uploadPartResult = - ossClient.uploadPart(uploadPartRequest); - statistics.incrementWriteOps(1); - partETags.add(uploadPartResult.getPartETag()); - } finally { - fis.close(); - } - } - CompleteMultipartUploadRequest completeMultipartUploadRequest = - new CompleteMultipartUploadRequest(bucketName, key, - uploadId, partETags); - CompleteMultipartUploadResult completeMultipartUploadResult = - ossClient.completeMultipartUpload(completeMultipartUploadRequest); - LOG.debug(completeMultipartUploadResult.getETag()); - } catch (OSSException | ClientException e) { - AbortMultipartUploadRequest abortMultipartUploadRequest = - new AbortMultipartUploadRequest(bucketName, key, uploadId); - ossClient.abortMultipartUpload(abortMultipartUploadRequest); - } - } - /** * list objects. * @@ -494,7 +422,7 @@ public void multipartUploadObject(String key, File file) throws IOException { * @return a list of matches. */ public ObjectListing listObjects(String prefix, int maxListingLength, - String marker, boolean recursive) { + String marker, boolean recursive) { String delimiter = recursive ? null : "/"; prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix); ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); @@ -605,7 +533,7 @@ public LocatedFileStatus next() throws IOException { if (hasNext()) { FileStatus status = batchIterator.next(); BlockLocation[] locations = fs.getFileBlockLocations(status, - 0, status.getLen()); + 0, status.getLen()); return new LocatedFileStatus( status, status.isFile() ? locations : null); } else { @@ -626,7 +554,7 @@ private boolean requestNextBatch() { List stats = new ArrayList<>( listing.getObjectSummaries().size() + listing.getCommonPrefixes().size()); - for(OSSObjectSummary summary: listing.getObjectSummaries()) { + for (OSSObjectSummary summary : listing.getObjectSummaries()) { String key = summary.getKey(); Path path = fs.makeQualified(new Path("/" + key)); if (filter.accept(path) && acceptor.accept(path, summary)) { @@ -637,7 +565,7 @@ private boolean requestNextBatch() { } } - for(String commonPrefix: listing.getCommonPrefixes()) { + for (String commonPrefix : listing.getCommonPrefixes()) { Path path = fs.makeQualified(new Path("/" + commonPrefix)); if (filter.accept(path) && acceptor.accept(path, commonPrefix)) { FileStatus status = new FileStatus(0, true, 1, 0, 0, path); @@ -656,4 +584,83 @@ private boolean requestNextBatch() { } }; } + + public PartETag uploadPart(File file, String key, String uploadId, int idx) + throws IOException { + InputStream instream = null; + Exception caught = null; + int tries = 3; + while (tries > 0) { + try { + instream = new FileInputStream(file); + UploadPartRequest uploadRequest = new UploadPartRequest(); + uploadRequest.setBucketName(bucketName); + uploadRequest.setKey(key); + uploadRequest.setUploadId(uploadId); + uploadRequest.setInputStream(instream); + uploadRequest.setPartSize(file.length()); + uploadRequest.setPartNumber(idx); + UploadPartResult uploadResult = ossClient.uploadPart(uploadRequest); + return uploadResult.getPartETag(); + } catch (Exception e) { + LOG.debug("Failed to upload "+ file.getPath() +", " + + "try again.", e); + caught = e; + } finally { + if (instream != null) { + instream.close(); + instream = null; + } + } + tries--; + } + + assert (caught != null); + throw new IOException("Failed to upload " + file.getPath() + + " for 3 times.", caught); + } + + /** + * Initiate multipart upload. + */ + public String getUploadId(String key) { + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, key); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + return initiateMultipartUploadResult.getUploadId(); + } + + /** + * Complete the specific multipart upload. + */ + public CompleteMultipartUploadResult completeMultipartUpload(String key, + String uploadId, List partETags) { + Collections.sort(partETags, new PartNumberAscendComparator()); + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, key, uploadId, + partETags); + return ossClient.completeMultipartUpload(completeMultipartUploadRequest); + } + + /** + * Abort the specific multipart upload. + */ + public void abortMultipartUpload(String key, String uploadId) { + AbortMultipartUploadRequest request = new AbortMultipartUploadRequest( + bucketName, key, uploadId); + ossClient.abortMultipartUpload(request); + } + + private static class PartNumberAscendComparator + implements Comparator, Serializable { + @Override + public int compare(PartETag o1, PartETag o2) { + if (o1.getPartNumber() > o2.getPartNumber()) { + return 1; + } else { + return -1; + } + } + } } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java deleted file mode 100644 index c75ee187bc9eb..0000000000000 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.aliyun.oss; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem.Statistics; -import org.apache.hadoop.fs.LocalDirAllocator; -import org.apache.hadoop.util.Progressable; - -import static org.apache.hadoop.fs.aliyun.oss.Constants.*; - -/** - * The output stream for OSS blob system. - * Data will be buffered on local disk, then uploaded to OSS in - * {@link #close()} method. - */ -public class AliyunOSSOutputStream extends OutputStream { - public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class); - private AliyunOSSFileSystemStore store; - private final String key; - private Statistics statistics; - private Progressable progress; - private long partSizeThreshold; - private LocalDirAllocator dirAlloc; - private boolean closed; - private File tmpFile; - private BufferedOutputStream backupStream; - - public AliyunOSSOutputStream(Configuration conf, - AliyunOSSFileSystemStore store, String key, Progressable progress, - Statistics statistics) throws IOException { - this.store = store; - this.key = key; - // The caller cann't get any progress information - this.progress = progress; - this.statistics = statistics; - partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, - MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); - - if (conf.get(BUFFER_DIR_KEY) == null) { - conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss"); - } - dirAlloc = new LocalDirAllocator(BUFFER_DIR_KEY); - - tmpFile = dirAlloc.createTmpFileForWrite("output-", - LocalDirAllocator.SIZE_UNKNOWN, conf); - backupStream = new BufferedOutputStream(new FileOutputStream(tmpFile)); - closed = false; - } - - @Override - public synchronized void close() throws IOException { - if (closed) { - return; - } - closed = true; - if (backupStream != null) { - backupStream.close(); - } - long dataLen = tmpFile.length(); - try { - if (dataLen <= partSizeThreshold) { - store.uploadObject(key, tmpFile); - } else { - store.multipartUploadObject(key, tmpFile); - } - } finally { - if (!tmpFile.delete()) { - LOG.warn("Can not delete file: " + tmpFile); - } - } - } - - - - @Override - public synchronized void flush() throws IOException { - backupStream.flush(); - } - - @Override - public synchronized void write(int b) throws IOException { - backupStream.write(b); - statistics.incrementBytesWritten(1); - } - -} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index 1a2160889a19c..2fe06c1b05fda 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -18,12 +18,14 @@ package org.apache.hadoop.fs.aliyun.oss; +import java.io.File; import java.io.IOException; -import java.io.InputStream; import com.aliyun.oss.common.auth.CredentialsProvider; +import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.security.ProviderUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,7 @@ final public class AliyunOSSUtils { private static final Logger LOG = LoggerFactory.getLogger(AliyunOSSUtils.class); + private static LocalDirAllocator directoryAllocator; private AliyunOSSUtils() { } @@ -74,31 +77,6 @@ public static String getValueWithKey(Configuration conf, String key) } } - /** - * Skip the requested number of bytes or fail if there are no enough bytes - * left. This allows for the possibility that {@link InputStream#skip(long)} - * may not skip as many bytes as requested (most likely because of reaching - * EOF). - * - * @param is the input stream to skip. - * @param n the number of bytes to skip. - * @throws IOException thrown when skipped less number of bytes. - */ - public static void skipFully(InputStream is, long n) throws IOException { - long total = 0; - long cur = 0; - - do { - cur = is.skip(n - total); - total += cur; - } while((total < n) && (cur > 0)); - - if (total < n) { - throw new IOException("Failed to skip " + n + " bytes, possibly due " + - "to EOF."); - } - } - /** * Calculate a proper size of multipart piece. If minPartSize * is too small, the number of multipart pieces may exceed the limit of @@ -126,7 +104,7 @@ public static CredentialsProvider getCredentialsProvider(Configuration conf) throws IOException { CredentialsProvider credentials; - String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY); + String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY); if (StringUtils.isEmpty(className)) { Configuration newConf = ProviderUtils.excludeIncompatibleCredentialProviders(conf, @@ -151,7 +129,7 @@ public static CredentialsProvider getCredentialsProvider(Configuration conf) throw new IOException(String.format("%s constructor exception. A " + "class specified in %s must provide an accessible constructor " + "accepting URI and Configuration, or an accessible default " + - "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), + "constructor.", className, CREDENTIALS_PROVIDER_KEY), e); } catch (ReflectiveOperationException | IllegalArgumentException e) { throw new IOException(className + " instantiation exception.", e); @@ -188,4 +166,85 @@ public static boolean objectRepresentsDirectory(final String name, final long size) { return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L; } + + /** + * Demand create the directory allocator, then create a temporary file. + * @param path prefix for the temporary file + * @param size the size of the file that is going to be written + * @param conf the Configuration object + * @return a unique temporary file + * @throws IOException IO problems + */ + public static File createTmpFileForWrite(String path, long size, + Configuration conf) throws IOException { + if (conf.get(BUFFER_DIR_KEY) == null) { + conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss"); + } + if (directoryAllocator == null) { + directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY); + } + return directoryAllocator.createTmpFileForWrite(path, size, conf); + } + + /** + * Get a integer option >= the minimum allowed value. + * @param conf configuration + * @param key key to look up + * @param defVal default value + * @param min minimum value + * @return the value + * @throws IllegalArgumentException if the value is below the minimum + */ + static int intOption(Configuration conf, String key, int defVal, int min) { + int v = conf.getInt(key, defVal); + Preconditions.checkArgument(v >= min, + String.format("Value of %s: %d is below the minimum value %d", + key, v, min)); + LOG.debug("Value of {} is {}", key, v); + return v; + } + + /** + * Get a long option >= the minimum allowed value. + * @param conf configuration + * @param key key to look up + * @param defVal default value + * @param min minimum value + * @return the value + * @throws IllegalArgumentException if the value is below the minimum + */ + static long longOption(Configuration conf, String key, long defVal, + long min) { + long v = conf.getLong(key, defVal); + Preconditions.checkArgument(v >= min, + String.format("Value of %s: %d is below the minimum value %d", + key, v, min)); + LOG.debug("Value of {} is {}", key, v); + return v; + } + + /** + * Get a size property from the configuration: this property must + * be at least equal to {@link Constants#MULTIPART_MIN_SIZE}. + * If it is too small, it is rounded up to that minimum, and a warning + * printed. + * @param conf configuration + * @param property property name + * @param defVal default value + * @return the value, guaranteed to be above the minimum size + */ + public static long getMultipartSizeProperty(Configuration conf, + String property, long defVal) { + long partSize = conf.getLong(property, defVal); + if (partSize < MULTIPART_MIN_SIZE) { + LOG.warn("{} must be at least 100 KB; configured value is {}", + property, partSize); + partSize = MULTIPART_MIN_SIZE; + } else if (partSize > Integer.MAX_VALUE) { + LOG.warn("oss: {} capped to ~2.14GB(maximum allowed size with " + + "current output mechanism)", MULTIPART_UPLOAD_PART_SIZE_KEY); + partSize = Integer.MAX_VALUE; + } + return partSize; + } } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 283927c828e0b..ecbd74905aaa3 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -31,10 +31,10 @@ private Constants() { // User agent public static final String USER_AGENT_PREFIX = "fs.oss.user.agent.prefix"; public static final String USER_AGENT_PREFIX_DEFAULT = - VersionInfoUtils.getDefaultUserAgent(); + VersionInfoUtils.getDefaultUserAgent(); // Class of credential provider - public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY = + public static final String CREDENTIALS_PROVIDER_KEY = "fs.oss.credentials.provider"; // OSS access verification @@ -82,10 +82,14 @@ private Constants() { public static final int MAX_PAGING_KEYS_DEFAULT = 1000; // Size of each of or multipart pieces in bytes - public static final String MULTIPART_UPLOAD_SIZE_KEY = + public static final String MULTIPART_UPLOAD_PART_SIZE_KEY = "fs.oss.multipart.upload.size"; + public static final long MULTIPART_UPLOAD_PART_SIZE_DEFAULT = + 104857600; // 100 MB + + /** The minimum multipart size which Aliyun OSS supports. */ + public static final int MULTIPART_MIN_SIZE = 100 * 1024; - public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024; public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000; // Minimum size in bytes before we start a multipart uploads or copy @@ -96,7 +100,6 @@ private Constants() { public static final String MULTIPART_DOWNLOAD_SIZE_KEY = "fs.oss.multipart.download.size"; - public static final long MULTIPART_DOWNLOAD_SIZE_DEFAULT = 512 * 1024; public static final String MULTIPART_DOWNLOAD_THREAD_NUMBER_KEY = @@ -139,9 +142,14 @@ private Constants() { public static final String FS_OSS_BLOCK_SIZE_KEY = "fs.oss.block.size"; public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024; + public static final String FS_OSS = "oss"; - public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L; - public static final int MAX_RETRIES = 10; + public static final String KEEPALIVE_TIME_KEY = + "fs.oss.threads.keepalivetime"; + public static final int KEEPALIVE_TIME_DEFAULT = 60; + public static final String UPLOAD_ACTIVE_BLOCKS_KEY = + "fs.oss.upload.active.blocks"; + public static final int UPLOAD_ACTIVE_BLOCKS_DEFAULT = 4; } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java similarity index 70% rename from hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java index 6b87d9ca46661..365d93142a37d 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java @@ -30,10 +30,13 @@ import java.io.IOException; +import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT; + /** - * Tests regular and multi-part upload functionality for AliyunOSSOutputStream. + * Tests regular and multi-part upload functionality for + * AliyunOSSBlockOutputStream. */ -public class TestAliyunOSSOutputStream { +public class TestAliyunOSSBlockOutputStream { private FileSystem fs; private static String testRootPath = AliyunOSSTestUtils.generateUniqueTestPath(); @@ -45,7 +48,7 @@ public class TestAliyunOSSOutputStream { public void setUp() throws Exception { Configuration conf = new Configuration(); conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024); - conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024); + conf.setInt(Constants.MULTIPART_UPLOAD_PART_SIZE_KEY, 5 * 1024 * 1024); fs = AliyunOSSTestUtils.createTestFileSystem(conf); } @@ -56,18 +59,39 @@ public void tearDown() throws Exception { } } - protected Path getTestPath() { + private Path getTestPath() { return new Path(testRootPath + "/test-aliyun-oss"); } + @Test + public void testZeroByteUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 0); + } + @Test public void testRegularUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024 - 1); ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024); + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024 + 1); } @Test public void testMultiPartUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), + 6 * 1024 * 1024 - 1); ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024); + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), + 6 * 1024 * 1024 + 1); + } + + @Test + public void testHugeUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), + MULTIPART_UPLOAD_PART_SIZE_DEFAULT - 1); + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), + MULTIPART_UPLOAD_PART_SIZE_DEFAULT); + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), + MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1); } @Test diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java index 66068c63621c7..32d0e464807d4 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java @@ -123,15 +123,15 @@ public void testSequentialAndRandomRead() throws Exception { + fsDataInputStream.getPos(), fsDataInputStream.getPos() == 0); assertTrue("expected position at:" - + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:" - + in.getExpectNextPos(), + + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:" + + in.getExpectNextPos(), in.getExpectNextPos() == Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT); fsDataInputStream.seek(4 * 1024 * 1024); assertTrue("expected position at:" + 4 * 1024 * 1024 - + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:" - + in.getExpectNextPos(), + + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:" + + in.getExpectNextPos(), in.getExpectNextPos() == 4 * 1024 * 1024 - + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT); + + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT); IOUtils.closeStream(fsDataInputStream); } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java index 18d09d5414955..e9a98b3307fdb 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java @@ -33,7 +33,7 @@ public class TestAliyunOSSContractDistCp extends AbstractContractDistCpTest { protected Configuration createConfiguration() { Configuration newConf = super.createConfiguration(); newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING); - newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING); + newConf.setLong(MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_SETTING); return newConf; } From 4ce45a8ec7b8109bc14ac47862f0ccbe7cdfa194 Mon Sep 17 00:00:00 2001 From: Konstantin V Shvachko Date: Fri, 30 Mar 2018 18:55:35 -0700 Subject: [PATCH 043/512] HADOOP-15253. Should update maxQueueSize when refresh call queue. Contributed by Tao Jie. (cherry picked from commit acfd764fcc9990e507c0e7cea746652375aaa632) --- .../main/java/org/apache/hadoop/ipc/Server.java | 3 +++ .../org/apache/hadoop/TestRefreshCallQueue.java | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index d0694fb6402db..c5da3b1809710 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -657,6 +657,9 @@ static Class getSchedulerClass( public synchronized void refreshCallQueue(Configuration conf) { // Create the next queue String prefix = getQueueClassPrefix(); + this.maxQueueSize = handlerCount * conf.getInt( + CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, + CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT); callQueue.swapQueue(getSchedulerClass(prefix, conf), getQueueClass(prefix, conf), maxQueueSize, prefix, conf); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java index d5eb9cfc4846c..faac1c0acaaf1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java @@ -30,8 +30,10 @@ import java.util.concurrent.LinkedBlockingQueue; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer; import org.apache.hadoop.hdfs.tools.DFSAdmin; @@ -141,8 +143,16 @@ public void testRefreshCallQueueWithFairCallQueue() throws Exception { // throw an error when we double-initialize JvmMetrics DefaultMetricsSystem.setMiniClusterMode(false); - + int serviceHandlerCount = config.getInt( + DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY, + DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT); NameNodeRpcServer rpcServer = (NameNodeRpcServer) cluster.getNameNodeRpc(); + // check callqueue size + assertEquals(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT + * serviceHandlerCount, rpcServer.getClientRpcServer().getMaxQueueSize()); + // Replace queue and update queue size + config.setInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, + 150); try { rpcServer.getClientRpcServer().refreshCallQueue(config); } catch (Exception e) { @@ -158,6 +168,9 @@ public void testRefreshCallQueueWithFairCallQueue() throws Exception { } finally { DefaultMetricsSystem.setMiniClusterMode(oldValue); } - } + // check callQueueSize has changed + assertEquals(150 * serviceHandlerCount, rpcServer.getClientRpcServer() + .getMaxQueueSize()); + } } \ No newline at end of file From 3c9983547cd878f4ad946804c0316de1637ca135 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Sat, 31 Mar 2018 09:46:13 -0700 Subject: [PATCH 044/512] HDFS-13289. RBF: TestConnectionManager#testCleanup() test case need correction. Contributed by Dibyendu Karmakar. (cherry picked from commit dc8e3432013153ac11d31d6b462aa96f8ca2c443) --- .../router/TestConnectionManager.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java index 741d1f6a2034e..2e4b80de10b6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java @@ -29,6 +29,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Test functionalities of {@link ConnectionManager}, which manages a pool @@ -94,14 +95,20 @@ public void testCleanup() throws Exception { // Make sure the number of connections doesn't go below minSize ConnectionPool pool3 = new ConnectionPool( conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10); - addConnectionsToPool(pool3, 10, 0); - poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool3); - connManager.cleanup(pool3); + addConnectionsToPool(pool3, 8, 0); + poolMap.put(new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS), pool3); + checkPoolConnections(TEST_USER3, 10, 0); + for (int i = 0; i < 10; i++) { + connManager.cleanup(pool3); + } checkPoolConnections(TEST_USER3, 2, 0); // With active connections added to pool, make sure it honors the // MIN_ACTIVE_RATIO again - addConnectionsToPool(pool3, 10, 2); - connManager.cleanup(pool3); + addConnectionsToPool(pool3, 8, 2); + checkPoolConnections(TEST_USER3, 10, 2); + for (int i = 0; i < 10; i++) { + connManager.cleanup(pool3); + } checkPoolConnections(TEST_USER3, 4, 2); } @@ -145,13 +152,18 @@ private void addConnectionsToPool(ConnectionPool pool, int numTotalConn, private void checkPoolConnections(UserGroupInformation ugi, int numOfConns, int numOfActiveConns) { + boolean connPoolFoundForUser = false; for (Map.Entry e : connManager.getPools().entrySet()) { if (e.getKey().getUgi() == ugi) { assertEquals(numOfConns, e.getValue().getNumConnections()); assertEquals(numOfActiveConns, e.getValue().getNumActiveConnections()); + connPoolFoundForUser = true; } } + if (!connPoolFoundForUser) { + fail("Connection pool not found for user " + ugi.getUserName()); + } } } From dcb2176c5dbbe48a258a9eb5437103bac37961ca Mon Sep 17 00:00:00 2001 From: Xiao Chen Date: Mon, 2 Apr 2018 22:47:02 -0700 Subject: [PATCH 045/512] HADOOP-15317. Improve NetworkTopology chooseRandom's loop. (cherry picked from commit 57374c4737ab0fccf52dae3cea911fc6bd90e1b7) --- .../apache/hadoop/net/NetworkTopology.java | 106 +++++++++++++++--- .../hadoop/net/TestNetworkTopology.java | 75 ++++++++++++- 2 files changed, 162 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index 278bf72f1b0be..256f07b696b3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -522,12 +522,12 @@ protected Node chooseRandom(final String scope, String excludedScope, numOfDatanodes -= ((InnerNode)node).getNumOfLeaves(); } } - if (numOfDatanodes == 0) { - LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\").", - scope, excludedScope); + if (numOfDatanodes <= 0) { + LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\")." + + " numOfDatanodes={}", + scope, excludedScope, numOfDatanodes); return null; } - Node ret = null; final int availableNodes; if (excludedScope == null) { availableNodes = countNumOfAvailableNodes(scope, excludedNodes); @@ -536,22 +536,96 @@ protected Node chooseRandom(final String scope, String excludedScope, countNumOfAvailableNodes("~" + excludedScope, excludedNodes); } LOG.debug("Choosing random from {} available nodes on node {}," - + " scope={}, excludedScope={}, excludeNodes={}", availableNodes, - innerNode, scope, excludedScope, excludedNodes); + + " scope={}, excludedScope={}, excludeNodes={}. numOfDatanodes={}.", + availableNodes, innerNode, scope, excludedScope, excludedNodes, + numOfDatanodes); + Node ret = null; if (availableNodes > 0) { - do { - int leaveIndex = r.nextInt(numOfDatanodes); - ret = innerNode.getLeaf(leaveIndex, node); - if (excludedNodes == null || !excludedNodes.contains(ret)) { + ret = chooseRandom(innerNode, node, excludedNodes, numOfDatanodes, + availableNodes); + } + LOG.debug("chooseRandom returning {}", ret); + return ret; + } + + /** + * Randomly choose one node under parentNode, considering the exclude + * nodes and scope. Should be called with {@link #netlock}'s readlock held. + * + * @param parentNode the parent node + * @param excludedScopeNode the node corresponding to the exclude scope. + * @param excludedNodes a collection of nodes to be excluded from + * @param totalInScopeNodes total number of nodes under parentNode, excluding + * the excludedScopeNode + * @param availableNodes number of available nodes under parentNode that + * could be chosen, excluding excludedNodes + * @return the chosen node, or null if none can be chosen + */ + private Node chooseRandom(final InnerNode parentNode, + final Node excludedScopeNode, final Collection excludedNodes, + final int totalInScopeNodes, final int availableNodes) { + Preconditions.checkArgument( + totalInScopeNodes >= availableNodes && availableNodes > 0, String + .format("%d should >= %d, and both should be positive.", + totalInScopeNodes, availableNodes)); + if (excludedNodes == null || excludedNodes.isEmpty()) { + // if there are no excludedNodes, randomly choose a node + final int index = r.nextInt(totalInScopeNodes); + return parentNode.getLeaf(index, excludedScopeNode); + } + + // excludedNodes non empty. + // Choose the nth VALID node, where n is random. VALID meaning it can be + // returned, after considering exclude scope and exclude nodes. + // The probability of being chosen should be equal for all VALID nodes. + // Notably, we do NOT choose nth node, and find the next valid node + // if n is excluded - this will make the probability of the node immediately + // after an excluded node higher. + // + // Start point is always 0 and that's fine, because the nth valid node + // logic provides equal randomness. + // + // Consider this example, where 1,3,5 out of the 10 nodes are excluded: + // 1 2 3 4 5 6 7 8 9 10 + // x x x + // We will randomly choose the nth valid node where n is [0,6]. + // We do NOT choose a random number n and just use the closest valid node, + // for example both n=3 and n=4 will choose 4, making it a 2/10 probability, + // higher than the expected 1/7 + // totalInScopeNodes=10 and availableNodes=7 in this example. + int nthValidToReturn = r.nextInt(availableNodes); + LOG.debug("nthValidToReturn is {}", nthValidToReturn); + Node ret = + parentNode.getLeaf(r.nextInt(totalInScopeNodes), excludedScopeNode); + if (!excludedNodes.contains(ret)) { + // return if we're lucky enough to get a valid node at a random first pick + LOG.debug("Chosen node {} from first random", ret); + return ret; + } else { + ret = null; + } + Node lastValidNode = null; + for (int i = 0; i < totalInScopeNodes; ++i) { + ret = parentNode.getLeaf(i, excludedScopeNode); + if (!excludedNodes.contains(ret)) { + if (nthValidToReturn == 0) { break; - } else { - LOG.debug("Node {} is excluded, continuing.", ret); } - // We've counted numOfAvailableNodes inside the lock, so there must be - // at least 1 satisfying node. Keep trying until we found it. - } while (true); + --nthValidToReturn; + lastValidNode = ret; + } else { + LOG.debug("Node {} is excluded, continuing.", ret); + ret = null; + } + } + if (ret == null && lastValidNode != null) { + LOG.error("BUG: Found lastValidNode {} but not nth valid node. " + + "parentNode={}, excludedScopeNode={}, excludedNodes={}, " + + "totalInScopeNodes={}, availableNodes={}, nthValidToReturn={}.", + lastValidNode, parentNode, excludedScopeNode, excludedNodes, + totalInScopeNodes, availableNodes, nthValidToReturn); + ret = lastValidNode; } - LOG.debug("chooseRandom returning {}", ret); return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java index 923a73f4c7255..f58f7c32680b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java @@ -27,10 +27,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Random; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -39,14 +38,19 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; public class TestNetworkTopology { - private static final Log LOG = LogFactory.getLog(TestNetworkTopology.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestNetworkTopology.class); private final static NetworkTopology cluster = NetworkTopology.getInstance(new Configuration()); private DatanodeDescriptor dataNodes[]; @@ -83,6 +87,7 @@ public void setupDatanodes() { } dataNodes[9].setDecommissioned(); dataNodes[10].setDecommissioned(); + GenericTestUtils.setLogLevel(NetworkTopology.LOG, Level.TRACE); } @Test @@ -324,6 +329,7 @@ private Map pickNodesAtRandom(int numNodes, frequency.put(random, frequency.get(random) + 1); } } + LOG.info("Result:" + frequency); return frequency; } @@ -471,4 +477,67 @@ public void testInvalidNetworkTopologiesNotCachedInHdfs() throws Exception { } } + /** + * Tests chooseRandom with include scope, excluding a few nodes. + */ + @Test + public void testChooseRandomInclude1() { + final String scope = "/d1"; + final Set excludedNodes = new HashSet<>(); + final Random r = new Random(); + for (int i = 0; i < 4; ++i) { + final int index = r.nextInt(5); + excludedNodes.add(dataNodes[index]); + } + Map frequency = pickNodesAtRandom(100, scope, excludedNodes); + + verifyResults(5, excludedNodes, frequency); + } + + /** + * Tests chooseRandom with include scope at rack, excluding a node. + */ + @Test + public void testChooseRandomInclude2() { + String scope = dataNodes[0].getNetworkLocation(); + Set excludedNodes = new HashSet<>(); + final Random r = new Random(); + int index = r.nextInt(1); + excludedNodes.add(dataNodes[index]); + final int count = 100; + Map frequency = + pickNodesAtRandom(count, scope, excludedNodes); + + verifyResults(1, excludedNodes, frequency); + } + + private void verifyResults(int upperbound, Set excludedNodes, + Map frequency) { + LOG.info("Excluded nodes are: {}", excludedNodes); + for (int i = 0; i < upperbound; ++i) { + final Node n = dataNodes[i]; + LOG.info("Verifying node {}", n); + if (excludedNodes.contains(n)) { + assertEquals(n + " should not have been chosen.", 0, + (int) frequency.get(n)); + } else { + assertTrue(n + " should have been chosen", frequency.get(n) > 0); + } + } + } + + /** + * Tests chooseRandom with include scope, no exlucde nodes. + */ + @Test + public void testChooseRandomInclude3() { + String scope = "/d1"; + Map frequency = pickNodesAtRandom(200, scope, null); + LOG.info("No node is excluded."); + for (int i = 0; i < 5; ++i) { + // all nodes should be more than zero + assertTrue(dataNodes[i] + " should have been chosen.", + frequency.get(dataNodes[i]) > 0); + } + } } From 8d755371f28b2212fb1446121019bd43267fc139 Mon Sep 17 00:00:00 2001 From: Xiao Chen Date: Mon, 2 Apr 2018 23:07:29 -0700 Subject: [PATCH 046/512] HADOOP-15355. TestCommonConfigurationFields is broken by HADOOP-15312. Contributed by LiXin Ge. (cherry picked from commit f614b44fe583033947cc78d8f04cfe6fb64fc892) --- .../apache/hadoop/crypto/key/KeyProvider.java | 11 ++++++---- .../fs/CommonConfigurationKeysPublic.java | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 62cc3818eb161..5d670e58810a8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -38,6 +38,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import javax.crypto.KeyGenerator; @@ -53,11 +54,13 @@ @InterfaceStability.Unstable public abstract class KeyProvider { public static final String DEFAULT_CIPHER_NAME = - "hadoop.security.key.default.cipher"; - public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding"; + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY; + public static final String DEFAULT_CIPHER = + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_DEFAULT_CIPHER_DEFAULT; public static final String DEFAULT_BITLENGTH_NAME = - "hadoop.security.key.default.bitlength"; - public static final int DEFAULT_BITLENGTH = 128; + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY; + public static final int DEFAULT_BITLENGTH = CommonConfigurationKeysPublic. + HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT; private final Configuration conf; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index bbc892ce22861..8cd753ab064f0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -678,6 +678,27 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_KEY_PROVIDER_PATH = "hadoop.security.key.provider.path"; + /** + * @see + * + * core-default.xml + */ + public static final String HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY = + "hadoop.security.key.default.bitlength"; + /** Defalt value for HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY. */ + public static final int HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT = 128; + + /** + * @see + * + * core-default.xml + */ + public static final String HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY = + "hadoop.security.key.default.cipher"; + /** Defalt value for HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY. */ + public static final String HADOOP_SECURITY_KEY_DEFAULT_CIPHER_DEFAULT = + "AES/CTR/NoPadding"; + // /** * @see From b89dc003026f303a0edf9c881e904e2279716fc5 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Tue, 3 Apr 2018 15:08:40 +0800 Subject: [PATCH 047/512] HDFS-13364. RBF: Support NamenodeProtocol in the Router. Contributed by Inigo Goiri. (cherry picked from commit 2be64eb201134502a92f7239bef8aa780771ca0b) --- .../federation/router/ConnectionContext.java | 35 +++- .../federation/router/ConnectionManager.java | 10 +- .../federation/router/ConnectionPool.java | 98 ++++++++- .../federation/router/ConnectionPoolId.java | 19 +- .../federation/router/RemoteMethod.java | 68 ++++++- .../router/RouterNamenodeProtocol.java | 187 ++++++++++++++++++ .../federation/router/RouterRpcClient.java | 56 ++++-- .../federation/router/RouterRpcServer.java | 111 ++++++++++- .../federation/MiniRouterDFSCluster.java | 8 + .../router/TestConnectionManager.java | 56 +++++- .../federation/router/TestRouterRpc.java | 115 +++++++++-- 11 files changed, 698 insertions(+), 65 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java index 1d27b51a167a5..7e779b5b1baf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import java.net.InetSocketAddress; + import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; -import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.ipc.RPC; /** @@ -26,18 +27,24 @@ * a connection, it increments a counter to mark it as active. Once the client * is done with the connection, it decreases the counter. It also takes care of * closing the connection once is not active. + * + * The protocols currently used are: + *

      + *
    • {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} + *
    • {@link org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol} + *
    */ public class ConnectionContext { /** Client for the connection. */ - private final ProxyAndInfo client; + private final ProxyAndInfo client; /** How many threads are using this connection. */ private int numThreads = 0; /** If the connection is closed. */ private boolean closed = false; - public ConnectionContext(ProxyAndInfo connection) { + public ConnectionContext(ProxyAndInfo connection) { this.client = connection; } @@ -74,7 +81,7 @@ public synchronized boolean isUsable() { * * @return Connection client. */ - public synchronized ProxyAndInfo getClient() { + public synchronized ProxyAndInfo getClient() { this.numThreads++; return this.client; } @@ -96,9 +103,27 @@ public synchronized void release() { public synchronized void close() { this.closed = true; if (this.numThreads == 0) { - ClientProtocol proxy = this.client.getProxy(); + Object proxy = this.client.getProxy(); // Nobody should be using this anymore so it should close right away RPC.stopProxy(proxy); } } + + @Override + public String toString() { + InetSocketAddress addr = this.client.getAddress(); + Object proxy = this.client.getProxy(); + Class clazz = proxy.getClass(); + + StringBuilder sb = new StringBuilder(); + sb.append(clazz.getSimpleName()); + sb.append("@"); + sb.append(addr); + sb.append("x"); + sb.append(numThreads); + if (closed) { + sb.append("[CLOSED]"); + } + return sb.toString(); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index 97c6403758d58..0b5084574e27a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -166,11 +166,12 @@ public void close() { * * @param ugi User group information. * @param nnAddress Namenode address for the connection. + * @param protocol Protocol for the connection. * @return Proxy client to connect to nnId as UGI. * @throws IOException If the connection cannot be obtained. */ - public ConnectionContext getConnection( - UserGroupInformation ugi, String nnAddress) throws IOException { + public ConnectionContext getConnection(UserGroupInformation ugi, + String nnAddress, Class protocol) throws IOException { // Check if the manager is shutdown if (!this.running) { @@ -181,7 +182,8 @@ public ConnectionContext getConnection( } // Try to get the pool if created - ConnectionPoolId connectionId = new ConnectionPoolId(ugi, nnAddress); + ConnectionPoolId connectionId = + new ConnectionPoolId(ugi, nnAddress, protocol); ConnectionPool pool = null; readLock.lock(); try { @@ -197,7 +199,7 @@ public ConnectionContext getConnection( pool = this.pools.get(connectionId); if (pool == null) { pool = new ConnectionPool( - this.conf, nnAddress, ugi, this.minSize, this.maxSize); + this.conf, nnAddress, ugi, this.minSize, this.maxSize, protocol); this.pools.put(connectionId, pool); } } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index 5af8a8679030a..6b416dd25e5e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -38,6 +38,9 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB; import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB; +import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryUtils; @@ -75,6 +78,8 @@ public class ConnectionPool { private final String namenodeAddress; /** User for this connections. */ private final UserGroupInformation ugi; + /** Class of the protocol. */ + private final Class protocol; /** Pool of connections. We mimic a COW array. */ private volatile List connections = new ArrayList<>(); @@ -91,16 +96,17 @@ public class ConnectionPool { protected ConnectionPool(Configuration config, String address, - UserGroupInformation user, int minPoolSize, int maxPoolSize) - throws IOException { + UserGroupInformation user, int minPoolSize, int maxPoolSize, + Class proto) throws IOException { this.conf = config; // Connection pool target this.ugi = user; this.namenodeAddress = address; + this.protocol = proto; this.connectionPoolId = - new ConnectionPoolId(this.ugi, this.namenodeAddress); + new ConnectionPoolId(this.ugi, this.namenodeAddress, this.protocol); // Set configuration parameters for the pool this.minSize = minPoolSize; @@ -287,7 +293,8 @@ public String getJSON() { * @throws IOException */ public ConnectionContext newConnection() throws IOException { - return newConnection(this.conf, this.namenodeAddress, this.ugi); + return newConnection( + this.conf, this.namenodeAddress, this.ugi, this.protocol); } /** @@ -299,12 +306,46 @@ public ConnectionContext newConnection() throws IOException { * @param conf Configuration for the connection. * @param nnAddress Address of server supporting the ClientProtocol. * @param ugi User context. - * @return Proxy for the target ClientProtocol that contains the user's + * @param proto Interface of the protocol. + * @return proto for the target ClientProtocol that contains the user's * security context. * @throws IOException If it cannot be created. */ protected static ConnectionContext newConnection(Configuration conf, - String nnAddress, UserGroupInformation ugi) + String nnAddress, UserGroupInformation ugi, Class proto) + throws IOException { + ConnectionContext ret; + if (proto == ClientProtocol.class) { + ret = newClientConnection(conf, nnAddress, ugi); + } else if (proto == NamenodeProtocol.class) { + ret = newNamenodeConnection(conf, nnAddress, ugi); + } else { + String msg = "Unsupported protocol for connection to NameNode: " + + ((proto != null) ? proto.getClass().getName() : "null"); + LOG.error(msg); + throw new IllegalStateException(msg); + } + return ret; + } + + /** + * Creates a proxy wrapper for a client NN connection. Each proxy contains + * context for a single user/security context. To maximize throughput it is + * recommended to use multiple connection per user+server, allowing multiple + * writes and reads to be dispatched in parallel. + * + * Mostly based on NameNodeProxies#createNonHAProxy() but it needs the + * connection identifier. + * + * @param conf Configuration for the connection. + * @param nnAddress Address of server supporting the ClientProtocol. + * @param ugi User context. + * @return Proxy for the target ClientProtocol that contains the user's + * security context. + * @throws IOException If it cannot be created. + */ + private static ConnectionContext newClientConnection( + Configuration conf, String nnAddress, UserGroupInformation ugi) throws IOException { RPC.setProtocolEngine( conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); @@ -334,4 +375,49 @@ protected static ConnectionContext newConnection(Configuration conf, ConnectionContext connection = new ConnectionContext(clientProxy); return connection; } + + /** + * Creates a proxy wrapper for a NN connection. Each proxy contains context + * for a single user/security context. To maximize throughput it is + * recommended to use multiple connection per user+server, allowing multiple + * writes and reads to be dispatched in parallel. + * + * @param conf Configuration for the connection. + * @param nnAddress Address of server supporting the ClientProtocol. + * @param ugi User context. + * @return Proxy for the target NamenodeProtocol that contains the user's + * security context. + * @throws IOException If it cannot be created. + */ + private static ConnectionContext newNamenodeConnection( + Configuration conf, String nnAddress, UserGroupInformation ugi) + throws IOException { + RPC.setProtocolEngine( + conf, NamenodeProtocolPB.class, ProtobufRpcEngine.class); + + final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy( + conf, + HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY, + HdfsClientConfigKeys.Retry.POLICY_ENABLED_DEFAULT, + HdfsClientConfigKeys.Retry.POLICY_SPEC_KEY, + HdfsClientConfigKeys.Retry.POLICY_SPEC_DEFAULT, + HdfsConstants.SAFEMODE_EXCEPTION_CLASS_NAME); + + SocketFactory factory = SocketFactory.getDefault(); + if (UserGroupInformation.isSecurityEnabled()) { + SaslRpcServer.init(conf); + } + InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress); + final long version = RPC.getProtocolVersion(NamenodeProtocolPB.class); + NamenodeProtocolPB proxy = RPC.getProtocolProxy(NamenodeProtocolPB.class, + version, socket, ugi, conf, + factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy(); + NamenodeProtocol client = new NamenodeProtocolTranslatorPB(proxy); + Text dtService = SecurityUtil.buildTokenService(socket); + + ProxyAndInfo clientProxy = + new ProxyAndInfo(client, dtService, socket); + ConnectionContext connection = new ConnectionContext(clientProxy); + return connection; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java index 6e1ee9a5c4da5..458fec203f774 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPoolId.java @@ -42,16 +42,21 @@ public class ConnectionPoolId implements Comparable { private final String nnId; /** Information about the user. */ private final UserGroupInformation ugi; + /** Protocol for the connection. */ + private final Class protocol; /** * New connection pool identifier. * * @param ugi Information of the user issuing the request. * @param nnId Namenode address with port. + * @param proto Protocol of the connection. */ - public ConnectionPoolId(final UserGroupInformation ugi, final String nnId) { + public ConnectionPoolId(final UserGroupInformation ugi, final String nnId, + final Class proto) { this.nnId = nnId; this.ugi = ugi; + this.protocol = proto; } @Override @@ -60,6 +65,7 @@ public int hashCode() { .append(this.nnId) .append(this.ugi.toString()) .append(this.getTokenIds()) + .append(this.protocol) .toHashCode(); return hash; } @@ -76,14 +82,18 @@ public boolean equals(Object o) { } String thisTokens = this.getTokenIds().toString(); String otherTokens = other.getTokenIds().toString(); - return thisTokens.equals(otherTokens); + if (!thisTokens.equals(otherTokens)) { + return false; + } + return this.protocol.equals(other.protocol); } return false; } @Override public String toString() { - return this.ugi + " " + this.getTokenIds() + "->" + this.nnId; + return this.ugi + " " + this.getTokenIds() + "->" + this.nnId + " [" + + this.protocol.getSimpleName() + "]"; } @Override @@ -97,6 +107,9 @@ public int compareTo(ConnectionPoolId other) { String otherTokens = other.getTokenIds().toString(); ret = thisTokens.compareTo(otherTokens); } + if (ret == 0) { + ret = this.protocol.toString().compareTo(other.protocol.toString()); + } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java index 7978105584ede..6ff2b01b0b679 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java @@ -38,22 +38,35 @@ public class RemoteMethod { private final Object[] params; /** List of method parameters types, matches parameters. */ private final Class[] types; + /** Class of the protocol for the method. */ + private final Class protocol; /** String name of the ClientProtocol method. */ private final String methodName; + /** + * Create a remote method generator for the ClientProtocol with no parameters. + * + * @param method The string name of the protocol method. + */ + public RemoteMethod(String method) { + this(ClientProtocol.class, method); + } + /** * Create a method with no parameters. * + * @param proto Protocol of the method. * @param method The string name of the ClientProtocol method. */ - public RemoteMethod(String method) { + public RemoteMethod(Class proto, String method) { this.params = null; this.types = null; this.methodName = method; + this.protocol = proto; } /** - * Creates a remote method generator. + * Create a remote method generator for the ClientProtocol. * * @param method The string name of the ClientProtocol method. * @param pTypes A list of types to use to locate the specific method. @@ -70,16 +83,49 @@ public RemoteMethod(String method) { */ public RemoteMethod(String method, Class[] pTypes, Object... pParams) throws IOException { + this(ClientProtocol.class, method, pTypes, pParams); + } + + /** + * Creates a remote method generator. + * + * @param proto Protocol of the method. + * @param method The string name of the ClientProtocol method. + * @param pTypes A list of types to use to locate the specific method. + * @param pParams A list of parameters for the method. The order of the + * parameter list must match the order and number of the types. + * Parameters are grouped into 2 categories: + *
      + *
    • Static parameters that are immutable across locations. + *
    • Dynamic parameters that are determined for each location by a + * RemoteParam object. To specify a dynamic parameter, pass an + * instance of RemoteParam in place of the parameter value. + *
    + * @throws IOException If the types and parameter lists are not valid. + */ + public RemoteMethod(Class proto, String method, Class[] pTypes, + Object... pParams) throws IOException { if (pParams.length != pTypes.length) { throw new IOException("Invalid parameters for method " + method); } + this.protocol = proto; this.params = pParams; this.types = Arrays.copyOf(pTypes, pTypes.length); this.methodName = method; } + /** + * Get the interface/protocol for this method. For example, ClientProtocol or + * NamenodeProtocol. + * + * @return Protocol for this method. + */ + public Class getProtocol() { + return this.protocol; + } + /** * Get the represented java method. * @@ -89,18 +135,18 @@ public RemoteMethod(String method, Class[] pTypes, Object... pParams) public Method getMethod() throws IOException { try { if (types != null) { - return ClientProtocol.class.getDeclaredMethod(methodName, types); + return protocol.getDeclaredMethod(methodName, types); } else { - return ClientProtocol.class.getDeclaredMethod(methodName); + return protocol.getDeclaredMethod(methodName); } } catch (NoSuchMethodException e) { // Re-throw as an IOException - LOG.error("Cannot get method {} with types {}", - methodName, Arrays.toString(types), e); + LOG.error("Cannot get method {} with types {} from {}", + methodName, Arrays.toString(types), protocol.getSimpleName(), e); throw new IOException(e); } catch (SecurityException e) { - LOG.error("Cannot access method {} with types {}", - methodName, Arrays.toString(types), e); + LOG.error("Cannot access method {} with types {} from {}", + methodName, Arrays.toString(types), protocol.getSimpleName(), e); throw new IOException(e); } } @@ -161,4 +207,10 @@ public Object[] getParams(RemoteLocationContext context) { } return objList; } + + @Override + public String toString() { + return this.protocol.getSimpleName() + "#" + this.methodName + " " + + Arrays.toString(this.params); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java new file mode 100644 index 0000000000000..0433650b94b3c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNamenodeProtocol.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; +import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; +import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; +import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; +import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; +import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; + +/** + * Module that implements all the RPC calls in {@link NamenodeProtocol} in the + * {@link RouterRpcServer}. + */ +public class RouterNamenodeProtocol implements NamenodeProtocol { + + /** RPC server to receive client calls. */ + private final RouterRpcServer rpcServer; + /** RPC clients to connect to the Namenodes. */ + private final RouterRpcClient rpcClient; + /** Interface to map global name space to HDFS subcluster name spaces. */ + private final FileSubclusterResolver subclusterResolver; + + + public RouterNamenodeProtocol(RouterRpcServer server) { + this.rpcServer = server; + this.rpcClient = this.rpcServer.getRPCClient(); + this.subclusterResolver = this.rpcServer.getSubclusterResolver(); + } + + @Override + public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, + long minBlockSize) throws IOException { + rpcServer.checkOperation(OperationCategory.READ); + + // Get the namespace where the datanode is located + Map map = + rpcServer.getDatanodeStorageReportMap(DatanodeReportType.ALL); + String nsId = null; + for (Entry entry : map.entrySet()) { + DatanodeStorageReport[] dns = entry.getValue(); + for (DatanodeStorageReport dn : dns) { + DatanodeInfo dnInfo = dn.getDatanodeInfo(); + if (dnInfo.getDatanodeUuid().equals(datanode.getDatanodeUuid())) { + nsId = entry.getKey(); + break; + } + } + // Break the loop if already found + if (nsId != null) { + break; + } + } + + // Forward to the proper namenode + if (nsId != null) { + RemoteMethod method = new RemoteMethod( + NamenodeProtocol.class, "getBlocks", + new Class[] {DatanodeInfo.class, long.class, long.class}, + datanode, size, minBlockSize); + return rpcClient.invokeSingle(nsId, method, BlocksWithLocations.class); + } + return null; + } + + @Override + public ExportedBlockKeys getBlockKeys() throws IOException { + rpcServer.checkOperation(OperationCategory.READ); + + // We return the information from the default name space + String defaultNsId = subclusterResolver.getDefaultNamespace(); + RemoteMethod method = + new RemoteMethod(NamenodeProtocol.class, "getBlockKeys"); + return rpcClient.invokeSingle(defaultNsId, method, ExportedBlockKeys.class); + } + + @Override + public long getTransactionID() throws IOException { + rpcServer.checkOperation(OperationCategory.READ); + + // We return the information from the default name space + String defaultNsId = subclusterResolver.getDefaultNamespace(); + RemoteMethod method = + new RemoteMethod(NamenodeProtocol.class, "getTransactionID"); + return rpcClient.invokeSingle(defaultNsId, method, long.class); + } + + @Override + public long getMostRecentCheckpointTxId() throws IOException { + rpcServer.checkOperation(OperationCategory.READ); + + // We return the information from the default name space + String defaultNsId = subclusterResolver.getDefaultNamespace(); + RemoteMethod method = + new RemoteMethod(NamenodeProtocol.class, "getMostRecentCheckpointTxId"); + return rpcClient.invokeSingle(defaultNsId, method, long.class); + } + + @Override + public CheckpointSignature rollEditLog() throws IOException { + rpcServer.checkOperation(OperationCategory.WRITE, false); + return null; + } + + @Override + public NamespaceInfo versionRequest() throws IOException { + rpcServer.checkOperation(OperationCategory.READ); + + // We return the information from the default name space + String defaultNsId = subclusterResolver.getDefaultNamespace(); + RemoteMethod method = + new RemoteMethod(NamenodeProtocol.class, "versionRequest"); + return rpcClient.invokeSingle(defaultNsId, method, NamespaceInfo.class); + } + + @Override + public void errorReport(NamenodeRegistration registration, int errorCode, + String msg) throws IOException { + rpcServer.checkOperation(OperationCategory.UNCHECKED, false); + } + + @Override + public NamenodeRegistration registerSubordinateNamenode( + NamenodeRegistration registration) throws IOException { + rpcServer.checkOperation(OperationCategory.WRITE, false); + return null; + } + + @Override + public NamenodeCommand startCheckpoint(NamenodeRegistration registration) + throws IOException { + rpcServer.checkOperation(OperationCategory.WRITE, false); + return null; + } + + @Override + public void endCheckpoint(NamenodeRegistration registration, + CheckpointSignature sig) throws IOException { + rpcServer.checkOperation(OperationCategory.WRITE, false); + } + + @Override + public RemoteEditLogManifest getEditLogManifest(long sinceTxId) + throws IOException { + rpcServer.checkOperation(OperationCategory.READ, false); + return null; + } + + @Override + public boolean isUpgradeFinalized() throws IOException { + rpcServer.checkOperation(OperationCategory.READ, false); + return false; + } + + @Override + public boolean isRollingUpgrade() throws IOException { + rpcServer.checkOperation(OperationCategory.READ, false); + return false; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index c973aa68779c1..ecb9f50832603 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -48,7 +48,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; -import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; @@ -225,14 +224,14 @@ public String getJSON() { * * @param ugi User group information. * @param nsId Nameservice identifier. - * @param rpcAddress ClientProtocol RPC server address of the NN. + * @param rpcAddress RPC server address of the NN. + * @param proto Protocol of the connection. * @return ConnectionContext containing a ClientProtocol proxy client for the * NN + current user. * @throws IOException If we cannot get a connection to the NameNode. */ - private ConnectionContext getConnection( - UserGroupInformation ugi, String nsId, String rpcAddress) - throws IOException { + private ConnectionContext getConnection(UserGroupInformation ugi, String nsId, + String rpcAddress, Class proto) throws IOException { ConnectionContext connection = null; try { // Each proxy holds the UGI info for the current user when it is created. @@ -242,7 +241,7 @@ private ConnectionContext getConnection( // for each individual request. // TODO Add tokens from the federated UGI - connection = this.connectionManager.getConnection(ugi, rpcAddress); + connection = this.connectionManager.getConnection(ugi, rpcAddress, proto); LOG.debug("User {} NN {} is using connection {}", ugi.getUserName(), rpcAddress, connection); } catch (Exception ex) { @@ -326,7 +325,8 @@ private RetryDecision shouldRetry(final IOException ioe, final int retryCount, private Object invokeMethod( final UserGroupInformation ugi, final List namenodes, - final Method method, final Object... params) throws IOException { + final Class protocol, final Method method, final Object... params) + throws IOException { if (namenodes == null || namenodes.isEmpty()) { throw new IOException("No namenodes to invoke " + method.getName() + @@ -344,9 +344,10 @@ private Object invokeMethod( try { String nsId = namenode.getNameserviceId(); String rpcAddress = namenode.getRpcAddress(); - connection = this.getConnection(ugi, nsId, rpcAddress); - ProxyAndInfo client = connection.getClient(); - ClientProtocol proxy = client.getProxy(); + connection = this.getConnection(ugi, nsId, rpcAddress, protocol); + ProxyAndInfo client = connection.getClient(); + final Object proxy = client.getProxy(); + ret = invoke(nsId, 0, method, proxy, params); if (failover) { // Success on alternate server, update @@ -611,7 +612,29 @@ public Object invokeSingle(final String nsId, RemoteMethod method) List nns = getNamenodesForNameservice(nsId); RemoteLocationContext loc = new RemoteLocation(nsId, "/"); - return invokeMethod(ugi, nns, method.getMethod(), method.getParams(loc)); + Class proto = method.getProtocol(); + Method m = method.getMethod(); + Object[] params = method.getParams(loc); + return invokeMethod(ugi, nns, proto, m, params); + } + + /** + * Invokes a remote method against the specified namespace. + * + * Re-throws exceptions generated by the remote RPC call as either + * RemoteException or IOException. + * + * @param nsId Target namespace for the method. + * @param method The remote method and parameters to invoke. + * @param clazz Class for the return type. + * @return The result of invoking the method. + * @throws IOException If the invoke generated an error. + */ + public T invokeSingle(final String nsId, RemoteMethod method, + Class clazz) throws IOException { + @SuppressWarnings("unchecked") + T ret = (T)invokeSingle(nsId, method); + return ret; } /** @@ -689,8 +712,9 @@ public T invokeSequential( List namenodes = getNamenodesForNameservice(ns); try { + Class proto = remoteMethod.getProtocol(); Object[] params = remoteMethod.getParams(loc); - Object result = invokeMethod(ugi, namenodes, m, params); + Object result = invokeMethod(ugi, namenodes, proto, m, params); // Check if the result is what we expected if (isExpectedClass(expectedResultClass, result) && isExpectedValue(expectedResultValue, result)) { @@ -914,8 +938,9 @@ public Map invokeConcurrent( String ns = location.getNameserviceId(); final List namenodes = getNamenodesForNameservice(ns); + Class proto = method.getProtocol(); Object[] paramList = method.getParams(location); - Object result = invokeMethod(ugi, namenodes, m, paramList); + Object result = invokeMethod(ugi, namenodes, proto, m, paramList); return Collections.singletonMap(location, clazz.cast(result)); } @@ -925,6 +950,7 @@ public Map invokeConcurrent( String nsId = location.getNameserviceId(); final List namenodes = getNamenodesForNameservice(nsId); + final Class proto = method.getProtocol(); final Object[] paramList = method.getParams(location); if (standby) { // Call the objectGetter to all NNs (including standby) @@ -939,7 +965,7 @@ public Map invokeConcurrent( orderedLocations.add(nnLocation); callables.add(new Callable() { public Object call() throws Exception { - return invokeMethod(ugi, nnList, m, paramList); + return invokeMethod(ugi, nnList, proto, m, paramList); } }); } @@ -948,7 +974,7 @@ public Object call() throws Exception { orderedLocations.add(location); callables.add(new Callable() { public Object call() throws Exception { - return invokeMethod(ugi, namenodes, m, paramList); + return invokeMethod(ugi, namenodes, proto, m, paramList); } }); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 383fd776bbf0c..11592894b41c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -101,9 +101,13 @@ import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol; +import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.NamenodeProtocolService; import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB; import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB; +import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; @@ -113,11 +117,18 @@ import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; +import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; +import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ProtobufRpcEngine; @@ -145,7 +156,8 @@ * the requests to the active * {@link org.apache.hadoop.hdfs.server.namenode.NameNode NameNode}. */ -public class RouterRpcServer extends AbstractService implements ClientProtocol { +public class RouterRpcServer extends AbstractService + implements ClientProtocol, NamenodeProtocol { private static final Logger LOG = LoggerFactory.getLogger(RouterRpcServer.class); @@ -191,6 +203,8 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol { private final Quota quotaCall; /** Erasure coding calls. */ private final ErasureCoding erasureCoding; + /** NamenodeProtocol calls. */ + private final RouterNamenodeProtocol nnProto; /** @@ -243,6 +257,11 @@ public RouterRpcServer(Configuration configuration, Router router, BlockingService clientNNPbService = ClientNamenodeProtocol .newReflectiveBlockingService(clientProtocolServerTranslator); + NamenodeProtocolServerSideTranslatorPB namenodeProtocolXlator = + new NamenodeProtocolServerSideTranslatorPB(this); + BlockingService nnPbService = NamenodeProtocolService + .newReflectiveBlockingService(namenodeProtocolXlator); + InetSocketAddress confRpcAddress = conf.getSocketAddr( RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY, RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY, @@ -261,6 +280,11 @@ public RouterRpcServer(Configuration configuration, Router router, .setQueueSizePerHandler(handlerQueueSize) .setVerbose(false) .build(); + + // Add all the RPC protocols that the Router implements + DFSUtil.addPBProtocol( + conf, NamenodeProtocolPB.class, nnPbService, this.rpcServer); + // We don't want the server to log the full stack trace for some exceptions this.rpcServer.addTerseExceptions( RemoteException.class, @@ -292,6 +316,7 @@ public RouterRpcServer(Configuration configuration, Router router, // Initialize modules this.quotaCall = new Quota(this.router, this); this.erasureCoding = new ErasureCoding(this); + this.nnProto = new RouterNamenodeProtocol(this); } @Override @@ -336,6 +361,15 @@ public RouterRpcClient getRPCClient() { return rpcClient; } + /** + * Get the subcluster resolver. + * + * @return Subcluster resolver. + */ + public FileSubclusterResolver getSubclusterResolver() { + return subclusterResolver; + } + /** * Get the RPC monitor and metrics. * @@ -1349,7 +1383,7 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) action, isChecked); Set nss = namenodeResolver.getNamespaces(); Map results = - rpcClient.invokeConcurrent(nss, method, true, true, boolean.class); + rpcClient.invokeConcurrent(nss, method, true, true, Boolean.class); // We only report true if all the name space are in safe mode int numSafemode = 0; @@ -1369,7 +1403,7 @@ public boolean restoreFailedStorage(String arg) throws IOException { new Class[] {String.class}, arg); final Set nss = namenodeResolver.getNamespaces(); Map ret = - rpcClient.invokeConcurrent(nss, method, true, false, boolean.class); + rpcClient.invokeConcurrent(nss, method, true, false, Boolean.class); boolean success = true; for (boolean s : ret.values()) { @@ -2070,6 +2104,77 @@ public BatchedEntries listOpenFiles(long prevId, return null; } + @Override // NamenodeProtocol + public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, + long minBlockSize) throws IOException { + return nnProto.getBlocks(datanode, size, minBlockSize); + } + + @Override // NamenodeProtocol + public ExportedBlockKeys getBlockKeys() throws IOException { + return nnProto.getBlockKeys(); + } + + @Override // NamenodeProtocol + public long getTransactionID() throws IOException { + return nnProto.getTransactionID(); + } + + @Override // NamenodeProtocol + public long getMostRecentCheckpointTxId() throws IOException { + return nnProto.getMostRecentCheckpointTxId(); + } + + @Override // NamenodeProtocol + public CheckpointSignature rollEditLog() throws IOException { + return nnProto.rollEditLog(); + } + + @Override // NamenodeProtocol + public NamespaceInfo versionRequest() throws IOException { + return nnProto.versionRequest(); + } + + @Override // NamenodeProtocol + public void errorReport(NamenodeRegistration registration, int errorCode, + String msg) throws IOException { + nnProto.errorReport(registration, errorCode, msg); + } + + @Override // NamenodeProtocol + public NamenodeRegistration registerSubordinateNamenode( + NamenodeRegistration registration) throws IOException { + return nnProto.registerSubordinateNamenode(registration); + } + + @Override // NamenodeProtocol + public NamenodeCommand startCheckpoint(NamenodeRegistration registration) + throws IOException { + return nnProto.startCheckpoint(registration); + } + + @Override // NamenodeProtocol + public void endCheckpoint(NamenodeRegistration registration, + CheckpointSignature sig) throws IOException { + nnProto.endCheckpoint(registration, sig); + } + + @Override // NamenodeProtocol + public RemoteEditLogManifest getEditLogManifest(long sinceTxId) + throws IOException { + return nnProto.getEditLogManifest(sinceTxId); + } + + @Override // NamenodeProtocol + public boolean isUpgradeFinalized() throws IOException { + return nnProto.isUpgradeFinalized(); + } + + @Override // NamenodeProtocol + public boolean isRollingUpgrade() throws IOException { + return nnProto.isRollingUpgrade(); + } + /** * Locate the location with the matching block pool id. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index c49f90a497e4c..0ad8536587b39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -239,6 +239,10 @@ public DFSClient getClient() throws IOException, URISyntaxException { } return client; } + + public Configuration getConf() { + return conf; + } } /** @@ -351,6 +355,10 @@ public String getConfSuffix() { } return suffix; } + + public Configuration getConf() { + return conf; + } } public MiniRouterDFSCluster( diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java index 2e4b80de10b6c..a7316481d7e85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.federation.router; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; @@ -68,14 +70,18 @@ public void testCleanup() throws Exception { Map poolMap = connManager.getPools(); ConnectionPool pool1 = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10); + conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, ClientProtocol.class); addConnectionsToPool(pool1, 9, 4); - poolMap.put(new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS), pool1); + poolMap.put( + new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class), + pool1); ConnectionPool pool2 = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER2, 0, 10); + conf, TEST_NN_ADDRESS, TEST_USER2, 0, 10, ClientProtocol.class); addConnectionsToPool(pool2, 10, 10); - poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool2); + poolMap.put( + new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS, ClientProtocol.class), + pool2); checkPoolConnections(TEST_USER1, 9, 4); checkPoolConnections(TEST_USER2, 10, 10); @@ -94,9 +100,11 @@ public void testCleanup() throws Exception { // Make sure the number of connections doesn't go below minSize ConnectionPool pool3 = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10); + conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10, ClientProtocol.class); addConnectionsToPool(pool3, 8, 0); - poolMap.put(new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS), pool3); + poolMap.put( + new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS, ClientProtocol.class), + pool3); checkPoolConnections(TEST_USER3, 10, 0); for (int i = 0; i < 10; i++) { connManager.cleanup(pool3); @@ -119,9 +127,41 @@ public void testGetConnection() throws Exception { int activeConns = 5; ConnectionPool pool = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10); + conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, ClientProtocol.class); addConnectionsToPool(pool, totalConns, activeConns); - poolMap.put(new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS), pool); + poolMap.put( + new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class), + pool); + + // All remaining connections should be usable + final int remainingSlots = totalConns - activeConns; + for (int i = 0; i < remainingSlots; i++) { + ConnectionContext cc = pool.getConnection(); + assertTrue(cc.isUsable()); + cc.getClient(); + activeConns++; + } + + checkPoolConnections(TEST_USER1, totalConns, activeConns); + + // Ask for more and this returns an active connection + ConnectionContext cc = pool.getConnection(); + assertTrue(cc.isActive()); + } + + @Test + public void getGetConnectionNamenodeProtocol() throws Exception { + Map poolMap = connManager.getPools(); + final int totalConns = 10; + int activeConns = 5; + + ConnectionPool pool = new ConnectionPool( + conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, NamenodeProtocol.class); + addConnectionsToPool(pool, totalConns, activeConns); + poolMap.put( + new ConnectionPoolId( + TEST_USER1, TEST_NN_ADDRESS, NamenodeProtocol.class), + pool); // All remaining connections should be usable final int remainingSlots = totalConns - activeConns; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index 50148806c9407..cc740989be377 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -59,6 +59,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; @@ -70,16 +71,22 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; +import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; +import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; @@ -133,6 +140,11 @@ public int compare( /** Client interface to the Namenode. */ private ClientProtocol nnProtocol; + /** NameNodeProtocol interface to the Router. */ + private NamenodeProtocol routerNamenodeProtocol; + /** NameNodeProtocol interface to the Namenode. */ + private NamenodeProtocol nnNamenodeProtocol; + /** Filesystem interface to the Router. */ private FileSystem routerFS; /** Filesystem interface to the Namenode. */ @@ -189,22 +201,18 @@ public void testSetup() throws Exception { // Wait to ensure NN has fully created its test directories Thread.sleep(100); - // Default namenode and random router for this test - this.router = cluster.getRandomRouter(); - this.ns = cluster.getNameservices().get(0); - this.namenode = cluster.getNamenode(ns, null); - - // Handles to the ClientProtocol interface - this.routerProtocol = router.getClient().getNamenode(); - this.nnProtocol = namenode.getClient().getNamenode(); + // Random router for this test + RouterContext rndRouter = cluster.getRandomRouter(); + this.setRouter(rndRouter); - // Handles to the filesystem client - this.nnFS = namenode.getFileSystem(); - this.routerFS = router.getFileSystem(); + // Pick a namenode for this test + String ns0 = cluster.getNameservices().get(0); + this.setNs(ns0); + this.setNamenode(cluster.getNamenode(ns0, null)); // Create a test file on the NN - Random r = new Random(); - String randomFile = "testfile-" + r.nextInt(); + Random rnd = new Random(); + String randomFile = "testfile-" + rnd.nextInt(); this.nnFile = cluster.getNamenodeTestDirectoryForNS(ns) + "/" + randomFile; this.routerFile = @@ -245,6 +253,8 @@ protected void setRouter(RouterContext r) this.router = r; this.routerProtocol = r.getClient().getNamenode(); this.routerFS = r.getFileSystem(); + this.routerNamenodeProtocol = NameNodeProxies.createProxy(router.getConf(), + router.getFileSystem().getUri(), NamenodeProtocol.class).getProxy(); } protected FileSystem getRouterFileSystem() { @@ -288,6 +298,12 @@ protected void setNamenode(NamenodeContext nn) this.namenode = nn; this.nnProtocol = nn.getClient().getNamenode(); this.nnFS = nn.getFileSystem(); + + // Namenode from the default namespace + String ns0 = cluster.getNameservices().get(0); + NamenodeContext nn0 = cluster.getNamenode(ns0, null); + this.nnNamenodeProtocol = NameNodeProxies.createProxy(nn0.getConf(), + nn0.getFileSystem().getUri(), NamenodeProtocol.class).getProxy(); } protected String getNs() { @@ -932,6 +948,79 @@ public void testProxyGetFileInfoAcessException() throws IOException { assertEquals(routerFailure.getClass(), nnFailure.getClass()); } + @Test + public void testProxyVersionRequest() throws Exception { + NamespaceInfo rVersion = routerNamenodeProtocol.versionRequest(); + NamespaceInfo nnVersion = nnNamenodeProtocol.versionRequest(); + assertEquals(nnVersion.getBlockPoolID(), rVersion.getBlockPoolID()); + assertEquals(nnVersion.getNamespaceID(), rVersion.getNamespaceID()); + assertEquals(nnVersion.getClusterID(), rVersion.getClusterID()); + assertEquals(nnVersion.getLayoutVersion(), rVersion.getLayoutVersion()); + assertEquals(nnVersion.getCTime(), rVersion.getCTime()); + } + + @Test + public void testProxyGetBlockKeys() throws Exception { + ExportedBlockKeys rKeys = routerNamenodeProtocol.getBlockKeys(); + ExportedBlockKeys nnKeys = nnNamenodeProtocol.getBlockKeys(); + assertEquals(nnKeys.getCurrentKey(), rKeys.getCurrentKey()); + assertEquals(nnKeys.getKeyUpdateInterval(), rKeys.getKeyUpdateInterval()); + assertEquals(nnKeys.getTokenLifetime(), rKeys.getTokenLifetime()); + } + + @Test + public void testProxyGetBlocks() throws Exception { + // Get datanodes + DatanodeInfo[] dns = + routerProtocol.getDatanodeReport(DatanodeReportType.ALL); + DatanodeInfo dn0 = dns[0]; + + // Verify that checking that datanode works + BlocksWithLocations routerBlockLocations = + routerNamenodeProtocol.getBlocks(dn0, 1024, 0); + BlocksWithLocations nnBlockLocations = + nnNamenodeProtocol.getBlocks(dn0, 1024, 0); + BlockWithLocations[] routerBlocks = routerBlockLocations.getBlocks(); + BlockWithLocations[] nnBlocks = nnBlockLocations.getBlocks(); + assertEquals(nnBlocks.length, routerBlocks.length); + for (int i = 0; i < routerBlocks.length; i++) { + assertEquals( + nnBlocks[i].getBlock().getBlockId(), + routerBlocks[i].getBlock().getBlockId()); + } + } + + @Test + public void testProxyGetTransactionID() throws IOException { + long routerTransactionID = routerNamenodeProtocol.getTransactionID(); + long nnTransactionID = nnNamenodeProtocol.getTransactionID(); + assertEquals(nnTransactionID, routerTransactionID); + } + + @Test + public void testProxyGetMostRecentCheckpointTxId() throws IOException { + long routerCheckPointId = + routerNamenodeProtocol.getMostRecentCheckpointTxId(); + long nnCheckPointId = nnNamenodeProtocol.getMostRecentCheckpointTxId(); + assertEquals(nnCheckPointId, routerCheckPointId); + } + + @Test + public void testProxySetSafemode() throws Exception { + boolean routerSafemode = + routerProtocol.setSafeMode(SafeModeAction.SAFEMODE_GET, false); + boolean nnSafemode = + nnProtocol.setSafeMode(SafeModeAction.SAFEMODE_GET, false); + assertEquals(nnSafemode, routerSafemode); + } + + @Test + public void testProxyRestoreFailedStorage() throws Exception { + boolean routerSuccess = routerProtocol.restoreFailedStorage("check"); + boolean nnSuccess = nnProtocol.restoreFailedStorage("check"); + assertEquals(nnSuccess, routerSuccess); + } + @Test public void testErasureCoding() throws IOException { From 78832eca8a2e7ee4a6d53694f05809ba4be384b9 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 19 Mar 2018 11:04:27 -0700 Subject: [PATCH 048/512] YARN-8002. Support NOT_SELF and ALL namespace types for allocation tag. (Weiwei Yang via wangda) Change-Id: I63b4e4192a95bf7ded98c54e46a2871c72869700 --- .../records/AllocationTagNamespaceType.java | 29 -- .../yarn/api/records/AllocationTags.java | 50 ---- .../api/resource/PlacementConstraints.java | 58 +++- .../InvalidAllocationTagException.java | 34 --- .../constraint}/AllocationTagNamespace.java | 114 +++----- .../scheduler/constraint/AllocationTags.java | 82 ++++++ .../constraint/AllocationTagsManager.java | 146 +++++++--- .../scheduler/constraint}/Evaluable.java | 2 +- .../constraint/PlacementConstraintsUtil.java | 55 +--- .../constraint}/TargetApplications.java | 8 +- .../algorithm/LocalAllocationTagsManager.java | 27 +- ...SingleConstraintAppPlacementAllocator.java | 17 +- .../yarn/server/resourcemanager/MockAM.java | 11 +- .../rmcontainer/TestRMContainerImpl.java | 47 ++- ...tSchedulingRequestContainerAllocation.java | 126 ++++++++ .../constraint/TestAllocationTagsManager.java | 272 +++++++++++++++--- .../TestAllocationTagsNamespace.java | 36 +-- .../TestPlacementConstraintsUtil.java | 247 +++++++++++++++- .../TestLocalAllocationTagsManager.java | 33 ++- ...SingleConstraintAppPlacementAllocator.java | 9 +- 20 files changed, 1031 insertions(+), 372 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTags.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidAllocationTagException.java rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records => hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint}/AllocationTagNamespace.java (75%) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTags.java rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records => hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint}/Evaluable.java (94%) rename hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records => hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint}/TargetApplications.java (92%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespaceType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespaceType.java index 5e46cd0f3d94c..de5492ee25f24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespaceType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespaceType.java @@ -18,12 +18,6 @@ package org.apache.hadoop.yarn.api.records; -import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; - -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; - /** * Class to describe all supported forms of namespaces for an allocation tag. */ @@ -44,29 +38,6 @@ public String getTypeKeyword() { return this.typeKeyword; } - /** - * Parses the namespace type from a given string. - * @param prefix namespace prefix. - * @return namespace type. - * @throws InvalidAllocationTagException - */ - public static AllocationTagNamespaceType fromString(String prefix) throws - InvalidAllocationTagException { - for (AllocationTagNamespaceType type : - AllocationTagNamespaceType.values()) { - if(type.getTypeKeyword().equals(prefix)) { - return type; - } - } - - Set values = Arrays.stream(AllocationTagNamespaceType.values()) - .map(AllocationTagNamespaceType::toString) - .collect(Collectors.toSet()); - throw new InvalidAllocationTagException( - "Invalid namespace prefix: " + prefix - + ", valid values are: " + String.join(",", values)); - } - @Override public String toString() { return this.getTypeKeyword(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTags.java deleted file mode 100644 index 50bffc3551615..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTags.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import java.util.Set; - -/** - * Allocation tags under same namespace. - */ -public class AllocationTags { - - private AllocationTagNamespace ns; - private Set tags; - - public AllocationTags(AllocationTagNamespace namespace, - Set allocationTags) { - this.ns = namespace; - this.tags = allocationTags; - } - - /** - * @return the namespace of these tags. - */ - public AllocationTagNamespace getNamespace() { - return this.ns; - } - - /** - * @return the allocation tags. - */ - public Set getTags() { - return this.tags; - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java index af70e2a7471d1..02138bd942932 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; +import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr; @@ -107,6 +107,25 @@ public static AbstractConstraint cardinality(String scope, int minCardinality, PlacementTargets.allocationTag(allocationTags)); } + /** + * Similar to {@link #cardinality(String, int, int, String...)}, but let you + * attach a namespace to the given allocation tags. + * + * @param scope the scope of the constraint + * @param namespace the namespace of the allocation tags + * @param minCardinality determines the minimum number of allocations within + * the scope + * @param maxCardinality determines the maximum number of allocations within + * the scope + * @param allocationTags allocation tags + * @return the resulting placement constraint + */ + public static AbstractConstraint cardinality(String scope, String namespace, + int minCardinality, int maxCardinality, String... allocationTags) { + return new SingleConstraint(scope, minCardinality, maxCardinality, + PlacementTargets.allocationTagWithNamespace(namespace, allocationTags)); + } + /** * Similar to {@link #cardinality(String, int, int, String...)}, but * determines only the minimum cardinality (the maximum cardinality is @@ -124,6 +143,23 @@ public static AbstractConstraint minCardinality(String scope, allocationTags); } + /** + * Similar to {@link #minCardinality(String, int, String...)}, but let you + * attach a namespace to the allocation tags. + * + * @param scope the scope of the constraint + * @param namespace the namespace of these tags + * @param minCardinality determines the minimum number of allocations within + * the scope + * @param allocationTags the constraint targets allocations with these tags + * @return the resulting placement constraint + */ + public static AbstractConstraint minCardinality(String scope, + String namespace, int minCardinality, String... allocationTags) { + return cardinality(scope, namespace, minCardinality, Integer.MAX_VALUE, + allocationTags); + } + /** * Similar to {@link #cardinality(String, int, int, String...)}, but * determines only the maximum cardinality (the minimum cardinality is 0). @@ -139,6 +175,23 @@ public static AbstractConstraint maxCardinality(String scope, return cardinality(scope, 0, maxCardinality, allocationTags); } + /** + * Similar to {@link #maxCardinality(String, int, String...)}, but let you + * specify a namespace for the tags, see supported namespaces in + * {@link AllocationTagNamespaceType}. + * + * @param scope the scope of the constraint + * @param tagNamespace the namespace of these tags + * @param maxCardinality determines the maximum number of allocations within + * the scope + * @param allocationTags allocation tags + * @return the resulting placement constraint + */ + public static AbstractConstraint maxCardinality(String scope, + String tagNamespace, int maxCardinality, String... allocationTags) { + return cardinality(scope, tagNamespace, 0, maxCardinality, allocationTags); + } + /** * This constraint generalizes the cardinality and target constraints. * @@ -242,9 +295,8 @@ public static TargetExpression allocationTagWithNamespace(String namespace, */ public static TargetExpression allocationTagToIntraApp( String... allocationTags) { - AllocationTagNamespace selfNs = new AllocationTagNamespace.Self(); return new TargetExpression(TargetType.ALLOCATION_TAG, - selfNs.toString(), allocationTags); + AllocationTagNamespaceType.SELF.toString(), allocationTags); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidAllocationTagException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidAllocationTagException.java deleted file mode 100644 index be8d881d7ae0b..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidAllocationTagException.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.exceptions; - -/** - * This exception is thrown by - * {@link - * org.apache.hadoop.yarn.api.records.AllocationTagNamespace#parse(String)} - * when it fails to parse a namespace. - */ -public class InvalidAllocationTagException extends YarnException { - - private static final long serialVersionUID = 1L; - - public InvalidAllocationTagException(String message) { - super(message); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagNamespace.java similarity index 75% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagNamespace.java index 25f876156bde6..7b9f3bee19b51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagNamespace.java @@ -16,22 +16,24 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.api.records; +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; +import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; +import org.apache.hadoop.yarn.api.records.ApplicationId; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.SELF; import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.NOT_SELF; import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.APP_LABEL; import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.APP_ID; import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.ALL; -import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.fromString; /** * Class to describe the namespace of an allocation tag. @@ -69,8 +71,6 @@ public AllocationTagNamespaceType getNamespaceType() { /** * Get the scope of the namespace, in form of a set of applications. - * Before calling this method, {@link #evaluate(TargetApplications)} - * must be called in prior to ensure the scope is proper evaluated. * * @return a set of applications. */ @@ -83,51 +83,20 @@ public Set getNamespaceScope() { return this.nsScope; } - @Override - public abstract void evaluate(TargetApplications target) - throws InvalidAllocationTagException; - /** - * @return true if the namespace is effective in all applications - * in this cluster. Specifically the namespace prefix should be - * "all". - */ - public boolean isGlobal() { - return AllocationTagNamespaceType.ALL.equals(getNamespaceType()); - } - - /** - * @return true if the namespace is effective within a single application - * by its application ID, the namespace prefix should be "app-id"; - * false otherwise. - */ - public boolean isSingleInterApp() { - return AllocationTagNamespaceType.APP_ID.equals(getNamespaceType()); - } - - /** - * @return true if the namespace is effective to the application itself, - * the namespace prefix should be "self"; false otherwise. - */ - public boolean isIntraApp() { - return AllocationTagNamespaceType.SELF.equals(getNamespaceType()); - } - - /** - * @return true if the namespace is effective to all applications except - * itself, the namespace prefix should be "not-self"; false otherwise. - */ - public boolean isNotSelf() { - return AllocationTagNamespaceType.NOT_SELF.equals(getNamespaceType()); - } - - /** - * @return true if the namespace is effective to a group of applications - * identified by a application label, the namespace prefix should be - * "app-label"; false otherwise. + * Evaluate the namespace against given target applications + * if it is necessary. Only self/not-self/app-label namespace types + * require this evaluation step, because they are not binding to a + * specific scope during initiating. So we do lazy binding for them + * in this method. + * + * @param target a generic type target that impacts this evaluation. + * @throws InvalidAllocationTagsQueryException */ - public boolean isAppLabel() { - return AllocationTagNamespaceType.APP_LABEL.equals(getNamespaceType()); + @Override + public void evaluate(TargetApplications target) + throws InvalidAllocationTagsQueryException { + // Sub-class needs to override this when it requires the eval step. } @Override @@ -146,9 +115,9 @@ public Self() { @Override public void evaluate(TargetApplications target) - throws InvalidAllocationTagException { + throws InvalidAllocationTagsQueryException { if (target == null || target.getCurrentApplicationId() == null) { - throw new InvalidAllocationTagException("Namespace Self must" + throw new InvalidAllocationTagsQueryException("Namespace Self must" + " be evaluated against a single application ID."); } ApplicationId applicationId = target.getCurrentApplicationId(); @@ -196,12 +165,6 @@ public static class All extends AllocationTagNamespace { public All() { super(ALL); } - - @Override - public void evaluate(TargetApplications target) { - Set allAppIds = target.getAllApplicationIds(); - setScopeIfNotNull(allAppIds); - } } /** @@ -229,10 +192,6 @@ public static class AppID extends AllocationTagNamespace { public AppID(ApplicationId applicationId) { super(APP_ID); this.targetAppId = applicationId; - } - - @Override - public void evaluate(TargetApplications target) { setScopeIfNotNull(ImmutableSet.of(targetAppId)); } @@ -248,11 +207,11 @@ public String toString() { * * @param namespaceStr namespace string. * @return an instance of {@link AllocationTagNamespace}. - * @throws InvalidAllocationTagException + * @throws InvalidAllocationTagsQueryException * if given string is not in valid format */ public static AllocationTagNamespace parse(String namespaceStr) - throws InvalidAllocationTagException { + throws InvalidAllocationTagsQueryException { // Return the default namespace if no valid string is given. if (Strings.isNullOrEmpty(namespaceStr)) { return new Self(); @@ -273,7 +232,7 @@ public static AllocationTagNamespace parse(String namespaceStr) return new All(); case APP_ID: if (nsValues.size() != 2) { - throw new InvalidAllocationTagException( + throw new InvalidAllocationTagsQueryException( "Missing the application ID in the namespace string: " + namespaceStr); } @@ -282,18 +241,35 @@ public static AllocationTagNamespace parse(String namespaceStr) case APP_LABEL: return new AppLabel(); default: - throw new InvalidAllocationTagException( + throw new InvalidAllocationTagsQueryException( "Invalid namespace string " + namespaceStr); } } + private static AllocationTagNamespaceType fromString(String prefix) throws + InvalidAllocationTagsQueryException { + for (AllocationTagNamespaceType type : + AllocationTagNamespaceType.values()) { + if(type.getTypeKeyword().equals(prefix)) { + return type; + } + } + + Set values = Arrays.stream(AllocationTagNamespaceType.values()) + .map(AllocationTagNamespaceType::toString) + .collect(Collectors.toSet()); + throw new InvalidAllocationTagsQueryException( + "Invalid namespace prefix: " + prefix + + ", valid values are: " + String.join(",", values)); + } + private static AllocationTagNamespace parseAppID(String appIDStr) - throws InvalidAllocationTagException { + throws InvalidAllocationTagsQueryException { try { ApplicationId applicationId = ApplicationId.fromString(appIDStr); return new AppID(applicationId); } catch (IllegalArgumentException e) { - throw new InvalidAllocationTagException( + throw new InvalidAllocationTagsQueryException( "Invalid application ID for " + APP_ID.getTypeKeyword() + ": " + appIDStr); } @@ -307,11 +283,11 @@ private static AllocationTagNamespace parseAppID(String appIDStr) * * @param namespaceStr namespace string. * @return a list of parsed strings. - * @throws InvalidAllocationTagException + * @throws InvalidAllocationTagsQueryException * if namespace format is unexpected. */ private static List normalize(String namespaceStr) - throws InvalidAllocationTagException { + throws InvalidAllocationTagsQueryException { List result = new ArrayList<>(); if (namespaceStr == null) { return result; @@ -326,7 +302,7 @@ private static List normalize(String namespaceStr) // Currently we only allow 1 or 2 values for a namespace string if (result.size() == 0 || result.size() > 2) { - throw new InvalidAllocationTagException("Invalid namespace string: " + throw new InvalidAllocationTagsQueryException("Invalid namespace string: " + namespaceStr + ", the syntax is or" + " /"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTags.java new file mode 100644 index 0000000000000..dc0237ec77b04 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTags.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +import java.util.Set; + +/** + * Allocation tags under same namespace. + */ +public final class AllocationTags { + + private AllocationTagNamespace ns; + private Set tags; + + private AllocationTags(AllocationTagNamespace namespace, + Set allocationTags) { + this.ns = namespace; + this.tags = allocationTags; + } + + /** + * @return the namespace of these tags. + */ + public AllocationTagNamespace getNamespace() { + return this.ns; + } + + /** + * @return the allocation tags. + */ + public Set getTags() { + return this.tags; + } + + @VisibleForTesting + public static AllocationTags createSingleAppAllocationTags( + ApplicationId appId, Set tags) { + AllocationTagNamespace namespace = new AllocationTagNamespace.AppID(appId); + return new AllocationTags(namespace, tags); + } + + @VisibleForTesting + public static AllocationTags createGlobalAllocationTags(Set tags) { + AllocationTagNamespace namespace = new AllocationTagNamespace.All(); + return new AllocationTags(namespace, tags); + } + + @VisibleForTesting + public static AllocationTags createOtherAppAllocationTags( + ApplicationId currentApp, Set allIds, Set tags) + throws InvalidAllocationTagsQueryException { + AllocationTagNamespace namespace = new AllocationTagNamespace.NotSelf(); + TargetApplications ta = new TargetApplications(currentApp, allIds); + namespace.evaluate(ta); + return new AllocationTags(namespace, tags); + } + + public static AllocationTags newAllocationTags( + AllocationTagNamespace namespace, Set tags) { + return new AllocationTags(namespace, tags); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index fb2619afcfa4b..830566a625a4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -22,9 +22,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -32,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -75,6 +78,12 @@ public static class TypeToCountedTags { // Map> private Map> typeToTagsWithCount = new HashMap<>(); + public TypeToCountedTags() {} + + private TypeToCountedTags(Map> tags) { + this.typeToTagsWithCount = tags; + } + // protected by external locks private void addTags(T type, Set tags) { Map innerMap = @@ -206,6 +215,52 @@ private boolean isEmpty() { public Map> getTypeToTagsWithCount() { return typeToTagsWithCount; } + + /** + * Absorbs the given {@link TypeToCountedTags} to current mapping, + * this will aggregate the count of the tags with same name. + * + * @param target a {@link TypeToCountedTags} to merge with. + */ + protected void absorb(final TypeToCountedTags target) { + // No opt if the given target is null. + if (target == null || target.getTypeToTagsWithCount() == null) { + return; + } + + // Merge the target. + Map> targetMap = target.getTypeToTagsWithCount(); + for (Map.Entry> targetEntry : + targetMap.entrySet()) { + // Get a mutable copy, do not modify the target reference. + Map copy = Maps.newHashMap(targetEntry.getValue()); + + // If the target type doesn't exist in the current mapping, + // add as a new entry. + Map existingMapping = + this.typeToTagsWithCount.putIfAbsent(targetEntry.getKey(), copy); + // There was a mapping for this target type, + // do proper merging on the operator. + if (existingMapping != null) { + Map localMap = + this.typeToTagsWithCount.get(targetEntry.getKey()); + // Merge the target map to the inner map. + Map targetValue = targetEntry.getValue(); + for (Map.Entry entry : targetValue.entrySet()) { + localMap.merge(entry.getKey(), entry.getValue(), + (a, b) -> Long.sum(a, b)); + } + } + } + } + + /** + * @return an immutable copy of current instance. + */ + protected TypeToCountedTags immutableCopy() { + return new TypeToCountedTags( + Collections.unmodifiableMap(this.typeToTagsWithCount)); + } } @VisibleForTesting @@ -235,6 +290,34 @@ public AllocationTagsManager(RMContext context) { rmContext = context; } + /** + * Aggregates multiple {@link TypeToCountedTags} to a single one based on + * a given set of application IDs, the values are properly merged. + * + * @param appIds a set of application IDs. + * @return an aggregated {@link TypeToCountedTags}. + */ + private TypeToCountedTags aggregateAllocationTags(Set appIds, + Map mapping) { + TypeToCountedTags result = new TypeToCountedTags(); + if (appIds != null) { + if (appIds.size() == 1) { + // If there is only one app, we simply return the mapping + // without any extra computation. + return mapping.get(appIds.iterator().next()); + } + + for (ApplicationId applicationId : appIds) { + TypeToCountedTags appIdTags = mapping.get(applicationId); + if (appIdTags != null) { + // Make sure ATM state won't be changed. + result.absorb(appIdTags.immutableCopy()); + } + } + } + return result; + } + /** * Notify container allocated on a node. * @@ -458,9 +541,8 @@ public boolean allocationTagExistsOnNode(NodeId nodeId, * to implement customized logic. * * @param nodeId nodeId, required. - * @param applicationId applicationId. When null is specified, return - * aggregated cardinality among all applications. - * @param tags allocation tags, see + * @param tags {@link AllocationTags}, allocation tags under a + * specific namespace. See * {@link SchedulingRequest#getAllocationTags()}, * When multiple tags specified. Returns cardinality * depends on op. If a specified tag doesn't exist, 0 @@ -474,29 +556,28 @@ public boolean allocationTagExistsOnNode(NodeId nodeId, * @throws InvalidAllocationTagsQueryException when illegal query * parameter specified */ - public long getNodeCardinalityByOp(NodeId nodeId, ApplicationId applicationId, - Set tags, LongBinaryOperator op) - throws InvalidAllocationTagsQueryException { + public long getNodeCardinalityByOp(NodeId nodeId, AllocationTags tags, + LongBinaryOperator op) throws InvalidAllocationTagsQueryException { readLock.lock(); - try { - if (nodeId == null || op == null) { + if (nodeId == null || op == null || tags == null) { throw new InvalidAllocationTagsQueryException( "Must specify nodeId/tags/op to query cardinality"); } TypeToCountedTags mapping; - if (applicationId != null) { - mapping = perAppNodeMappings.get(applicationId); - } else { + if (AllocationTagNamespaceType.ALL.equals( + tags.getNamespace().getNamespaceType())) { mapping = globalNodeMapping; + } else { + // Aggregate app tags cardinality by applications. + mapping = aggregateAllocationTags( + tags.getNamespace().getNamespaceScope(), + perAppNodeMappings); } - if (mapping == null) { - return 0; - } - - return mapping.getCardinality(nodeId, tags, op); + return mapping == null ? 0 : + mapping.getCardinality(nodeId, tags.getTags(), op); } finally { readLock.unlock(); } @@ -507,9 +588,8 @@ public long getNodeCardinalityByOp(NodeId nodeId, ApplicationId applicationId, * to implement customized logic. * * @param rack rack, required. - * @param applicationId applicationId. When null is specified, return - * aggregated cardinality among all applications. - * @param tags allocation tags, see + * @param tags {@link AllocationTags}, allocation tags under a + * specific namespace. See * {@link SchedulingRequest#getAllocationTags()}, * When multiple tags specified. Returns cardinality * depends on op. If a specified tag doesn't exist, 0 @@ -523,30 +603,28 @@ public long getNodeCardinalityByOp(NodeId nodeId, ApplicationId applicationId, * @throws InvalidAllocationTagsQueryException when illegal query * parameter specified */ - @SuppressWarnings("unchecked") - public long getRackCardinalityByOp(String rack, ApplicationId applicationId, - Set tags, LongBinaryOperator op) - throws InvalidAllocationTagsQueryException { + public long getRackCardinalityByOp(String rack, AllocationTags tags, + LongBinaryOperator op) throws InvalidAllocationTagsQueryException { readLock.lock(); - try { - if (rack == null || op == null) { + if (rack == null || op == null || tags == null) { throw new InvalidAllocationTagsQueryException( - "Must specify rack/tags/op to query cardinality"); + "Must specify nodeId/tags/op to query cardinality"); } TypeToCountedTags mapping; - if (applicationId != null) { - mapping = perAppRackMappings.get(applicationId); - } else { + if (AllocationTagNamespaceType.ALL.equals( + tags.getNamespace().getNamespaceType())) { mapping = globalRackMapping; + } else { + // Aggregates cardinality by rack. + mapping = aggregateAllocationTags( + tags.getNamespace().getNamespaceScope(), + perAppRackMappings); } - if (mapping == null) { - return 0; - } - - return mapping.getCardinality(rack, tags, op); + return mapping == null ? 0 : + mapping.getCardinality(rack, tags.getTags(), op); } finally { readLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Evaluable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/Evaluable.java similarity index 94% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Evaluable.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/Evaluable.java index 7a74002ea5d70..6a7e54e470814 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Evaluable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/Evaluable.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.api.records; +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import org.apache.hadoop.yarn.exceptions.YarnException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index 2d0e95a9b9f51..389fc5cc908c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -24,11 +24,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; -import org.apache.hadoop.yarn.api.records.TargetApplications; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; @@ -38,7 +36,6 @@ import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; -import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm; @@ -70,43 +67,25 @@ private PlacementConstraintsUtil() { */ private static AllocationTagNamespace getAllocationTagNamespace( ApplicationId currentAppId, String targetKey, AllocationTagsManager atm) - throws InvalidAllocationTagException{ + throws InvalidAllocationTagsQueryException { // Parse to a valid namespace. AllocationTagNamespace namespace = AllocationTagNamespace.parse(targetKey); - // TODO remove such check once we support all forms of namespaces - if (!namespace.isIntraApp() && !namespace.isSingleInterApp()) { - throw new InvalidAllocationTagException( - "Only support " + AllocationTagNamespaceType.SELF.toString() - + " and "+ AllocationTagNamespaceType.APP_ID + " now," - + namespace.toString() + " is not supported yet!"); + // TODO Complete remove this check once we support app-label. + if (AllocationTagNamespaceType.APP_LABEL + .equals(namespace.getNamespaceType())) { + throw new InvalidAllocationTagsQueryException( + namespace.toString() + " is not supported yet!"); } // Evaluate the namespace according to the given target // before it can be consumed. - TargetApplications ta = new TargetApplications(currentAppId, - atm.getAllApplicationIds()); + TargetApplications ta = + new TargetApplications(currentAppId, atm.getAllApplicationIds()); namespace.evaluate(ta); return namespace; } - // We return a single app Id now, because at present, - // only self and app-id namespace is supported. But moving on, - // this will return a set of application IDs. - // TODO support other forms of namespaces - private static ApplicationId getNamespaceScope( - AllocationTagNamespace namespace) - throws InvalidAllocationTagException { - if (namespace.getNamespaceScope() == null - || namespace.getNamespaceScope().size() != 1) { - throw new InvalidAllocationTagException( - "Invalid allocation tag namespace " + namespace.toString() - + ", expecting it is not null and only 1 application" - + " ID in the scope."); - } - return namespace.getNamespaceScope().iterator().next(); - } - /** * Returns true if single placement constraint with associated * allocationTags and scope is satisfied by a specific scheduler Node. @@ -128,14 +107,10 @@ private static boolean canSatisfySingleConstraintExpression( // Parse the allocation tag's namespace from the given target key, // then evaluate the namespace and get its scope, // which is represented by one or more application IDs. - ApplicationId effectiveAppID; - try { - AllocationTagNamespace namespace = getAllocationTagNamespace( + AllocationTagNamespace namespace = getAllocationTagNamespace( targetApplicationId, te.getTargetKey(), tm); - effectiveAppID = getNamespaceScope(namespace); - } catch (InvalidAllocationTagException e) { - throw new InvalidAllocationTagsQueryException(e); - } + AllocationTags allocationTags = AllocationTags + .newAllocationTags(namespace, te.getTargetValues()); long minScopeCardinality = 0; long maxScopeCardinality = 0; @@ -149,20 +124,20 @@ private static boolean canSatisfySingleConstraintExpression( if (sc.getScope().equals(PlacementConstraints.NODE)) { if (checkMinCardinality) { minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), - effectiveAppID, te.getTargetValues(), Long::max); + allocationTags, Long::max); } if (checkMaxCardinality) { maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), - effectiveAppID, te.getTargetValues(), Long::min); + allocationTags, Long::min); } } else if (sc.getScope().equals(PlacementConstraints.RACK)) { if (checkMinCardinality) { minScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(), - effectiveAppID, te.getTargetValues(), Long::max); + allocationTags, Long::max); } if (checkMaxCardinality) { maxScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(), - effectiveAppID, te.getTargetValues(), Long::min); + allocationTags, Long::min); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/TargetApplications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TargetApplications.java similarity index 92% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/TargetApplications.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TargetApplications.java index de0ea268b1039..0de7c9ec6b100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/TargetApplications.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TargetApplications.java @@ -16,7 +16,9 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.api.records; +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; + +import org.apache.hadoop.yarn.api.records.ApplicationId; import java.util.Set; import java.util.stream.Collectors; @@ -37,10 +39,6 @@ public TargetApplications(ApplicationId currentApplicationId, this.allAppIds = allApplicationIds; } - public Set getAllApplicationIds() { - return this.allAppIds; - } - public ApplicationId getCurrentApplicationId() { return this.currentAppId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/LocalAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/LocalAllocationTagsManager.java index 9472719ae6d82..1fce466d3ac83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/LocalAllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/LocalAllocationTagsManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -139,29 +140,27 @@ public long getNodeCardinality(NodeId nodeId, ApplicationId applicationId, } @Override - public long getRackCardinality(String rack, ApplicationId applicationId, - String tag) throws InvalidAllocationTagsQueryException { - return tagsManager.getRackCardinality(rack, applicationId, tag); + public long getNodeCardinalityByOp(NodeId nodeId, AllocationTags tags, + LongBinaryOperator op) throws InvalidAllocationTagsQueryException { + return tagsManager.getNodeCardinalityByOp(nodeId, tags, op); } @Override - public boolean allocationTagExistsOnNode(NodeId nodeId, - ApplicationId applicationId, String tag) - throws InvalidAllocationTagsQueryException { - return tagsManager.allocationTagExistsOnNode(nodeId, applicationId, tag); + public long getRackCardinality(String rack, ApplicationId applicationId, + String tag) throws InvalidAllocationTagsQueryException { + return tagsManager.getRackCardinality(rack, applicationId, tag); } @Override - public long getNodeCardinalityByOp(NodeId nodeId, - ApplicationId applicationId, Set tags, LongBinaryOperator op) - throws InvalidAllocationTagsQueryException { - return tagsManager.getNodeCardinalityByOp(nodeId, applicationId, tags, op); + public long getRackCardinalityByOp(String rack, AllocationTags tags, + LongBinaryOperator op) throws InvalidAllocationTagsQueryException { + return tagsManager.getRackCardinalityByOp(rack, tags, op); } @Override - public long getRackCardinalityByOp(String rack, ApplicationId applicationId, - Set tags, LongBinaryOperator op) + public boolean allocationTagExistsOnNode(NodeId nodeId, + ApplicationId applicationId, String tag) throws InvalidAllocationTagsQueryException { - return tagsManager.getRackCardinalityByOp(rack, applicationId, tags, op); + return tagsManager.allocationTagExistsOnNode(nodeId, applicationId, tag); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 7e5506efddc04..9004110837e74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -23,7 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagNamespace; import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -32,7 +32,6 @@ import org.apache.hadoop.yarn.api.records.impl.pb.SchedulingRequestPBImpl; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; -import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -339,18 +338,18 @@ private void validateAndSetSchedulingRequest(SchedulingRequest try { AllocationTagNamespace tagNS = AllocationTagNamespace.parse(targetExpression.getTargetKey()); - if (!AllocationTagNamespaceType.SELF + if (AllocationTagNamespaceType.APP_LABEL .equals(tagNS.getNamespaceType())) { throwExceptionWithMetaInfo( - "As of now, the only accepted target key for targetKey of " - + "allocation_tag target expression is: [" - + AllocationTagNamespaceType.SELF.toString() - + "]. Please make changes to placement constraints " - + "accordingly. If this is null, it will be set to " + "As of now, allocation tag namespace [" + + AllocationTagNamespaceType.APP_LABEL.toString() + + "] is not supported. Please make changes to placement " + + "constraints accordingly. If this is null, it will be " + + "set to " + AllocationTagNamespaceType.SELF.toString() + " by default."); } - } catch (InvalidAllocationTagException e) { + } catch (InvalidAllocationTagsQueryException e) { throwExceptionWithMetaInfo( "Invalid allocation tag namespace, message: " + e.getMessage()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index 2ed201ce91875..5eb667e144bac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -305,6 +305,14 @@ public AllocateResponse allocate(List resourceRequest, public AllocateResponse allocateIntraAppAntiAffinity( ResourceSizing resourceSizing, Priority priority, long allocationId, Set allocationTags, String... targetTags) throws Exception { + return allocateAppAntiAffinity(resourceSizing, priority, allocationId, + null, allocationTags, targetTags); + } + + public AllocateResponse allocateAppAntiAffinity( + ResourceSizing resourceSizing, Priority priority, long allocationId, + String namespace, Set allocationTags, String... targetTags) + throws Exception { return this.allocate(null, Arrays.asList(SchedulingRequest.newBuilder().executionType( ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)) @@ -313,7 +321,8 @@ public AllocateResponse allocateIntraAppAntiAffinity( PlacementConstraints .targetNotIn(PlacementConstraints.NODE, PlacementConstraints.PlacementTargets - .allocationTagToIntraApp(targetTags)).build()) + .allocationTagWithNamespace(namespace, targetTags)) + .build()) .resourceSizing(resourceSizing).build()), null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 27c5fbdb1f3ce..7a930cd361159 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -63,6 +63,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -428,20 +430,27 @@ public void testContainerTransitionNotifyAllocationTagsManager() rmContainer.setAllocationTags(ImmutableSet.of("mapper")); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), null), + Long::max)); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); Assert.assertEquals(1, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); rmContainer.handle(new RMContainerFinishedEvent(containerId, ContainerStatus .newInstance(containerId, ContainerState.COMPLETE, "", 0), RMContainerEventType.KILL)); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); /* Second container: ACQUIRED -> FINISHED */ rmContainer = new RMContainerImpl(container, @@ -449,14 +458,18 @@ public void testContainerTransitionNotifyAllocationTagsManager() nodeId, "user", rmContext); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); rmContainer.setAllocationTags(ImmutableSet.of("mapper")); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); Assert.assertEquals(1, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); rmContainer.handle( new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); @@ -466,7 +479,9 @@ public void testContainerTransitionNotifyAllocationTagsManager() RMContainerEventType.FINISHED)); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); /* Third container: RUNNING -> FINISHED */ rmContainer = new RMContainerImpl(container, @@ -475,13 +490,17 @@ public void testContainerTransitionNotifyAllocationTagsManager() rmContainer.setAllocationTags(ImmutableSet.of("mapper")); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); Assert.assertEquals(1, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); rmContainer.handle( new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); @@ -494,7 +513,9 @@ public void testContainerTransitionNotifyAllocationTagsManager() RMContainerEventType.FINISHED)); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); /* Fourth container: NEW -> RECOVERED */ rmContainer = new RMContainerImpl(container, @@ -503,7 +524,9 @@ public void testContainerTransitionNotifyAllocationTagsManager() rmContainer.setAllocationTags(ImmutableSet.of("mapper")); Assert.assertEquals(0, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); NMContainerStatus containerStatus = NMContainerStatus .newInstance(containerId, 0, ContainerState.NEW, @@ -514,6 +537,8 @@ public void testContainerTransitionNotifyAllocationTagsManager() .handle(new RMContainerRecoverEvent(containerId, containerStatus)); Assert.assertEquals(1, - tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + tagsManager.getNodeCardinalityByOp(nodeId, + AllocationTags.createSingleAppAllocationTags(appId, null), + Long::max)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java index 27d86611e3171..d7124bb7a9c6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagNamespace; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceSizing; @@ -224,6 +225,131 @@ public RMNodeLabelsManager createNodeLabelManager() { rm1.close(); } + /** + * This UT covers some basic end-to-end inter-app anti-affinity + * constraint tests. For comprehensive tests over different namespace + * types, see more in TestPlacementConstraintsUtil. + * @throws Exception + */ + @Test + public void testInterAppAntiAffinity() throws Exception { + Configuration csConf = TestUtils.getConfigurationWithMultipleQueues( + new Configuration()); + csConf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, + YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); + + // inject node label manager + MockRM rm1 = new MockRM(csConf) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + + rm1.getRMContext().setNodeLabelManager(mgr); + rm1.start(); + + // 4 NMs. + MockNM[] nms = new MockNM[4]; + RMNode[] rmNodes = new RMNode[4]; + for (int i = 0; i < 4; i++) { + nms[i] = rm1.registerNode("192.168.0." + i + ":1234", 10 * GB); + rmNodes[i] = rm1.getRMContext().getRMNodes().get(nms[i].getNodeId()); + } + + // app1 -> c + RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nms[0]); + + // app1 asks for 3 anti-affinity containers for the same app. It should + // only get 3 containers allocated to 3 different nodes.. + am1.allocateIntraAppAntiAffinity( + ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)), + Priority.newInstance(1), 1L, ImmutableSet.of("mapper"), "mapper"); + + CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNodes[j])); + } + } + + System.out.println("Mappers on HOST0: " + + rmNodes[0].getAllocationTagsWithCount().get("mapper")); + System.out.println("Mappers on HOST1: " + + rmNodes[1].getAllocationTagsWithCount().get("mapper")); + System.out.println("Mappers on HOST2: " + + rmNodes[2].getAllocationTagsWithCount().get("mapper")); + + // App1 should get 4 containers allocated (1 AM + 3 mappers). + FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt( + am1.getApplicationAttemptId()); + Assert.assertEquals(4, schedulerApp.getLiveContainers().size()); + + // app2 -> c + RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nms[0]); + + // App2 asks for 3 containers that anti-affinity with any mapper, + // since 3 out of 4 nodes already have mapper containers, all 3 + // containers will be allocated on the other node. + AllocationTagNamespace.All allNs = new AllocationTagNamespace.All(); + am2.allocateAppAntiAffinity( + ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)), + Priority.newInstance(1), 1L, allNs.toString(), + ImmutableSet.of("foo"), "mapper"); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNodes[j])); + } + } + + FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt( + am2.getApplicationAttemptId()); + + // App2 should get 4 containers allocated (1 AM + 3 container). + Assert.assertEquals(4, schedulerApp2.getLiveContainers().size()); + + // The allocated node should not have mapper tag. + Assert.assertTrue(schedulerApp2.getLiveContainers() + .stream().allMatch(rmContainer -> { + // except the nm host + if (!rmContainer.getContainer().getNodeId().equals(rmNodes[0])) { + return !rmContainer.getAllocationTags().contains("mapper"); + } + return true; + })); + + // app3 -> c + RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "c"); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nms[0]); + + // App3 asks for 3 containers that anti-affinity with any mapper. + // Unlike the former case, since app3 source tags are also mapper, + // it will anti-affinity with itself too. So there will be only 1 + // container be allocated. + am3.allocateAppAntiAffinity( + ResourceSizing.newInstance(3, Resource.newInstance(1024, 1)), + Priority.newInstance(1), 1L, allNs.toString(), + ImmutableSet.of("mapper"), "mapper"); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 4; j++) { + cs.handle(new NodeUpdateSchedulerEvent(rmNodes[j])); + } + } + + FiCaSchedulerApp schedulerApp3 = cs.getApplicationAttempt( + am3.getApplicationAttemptId()); + + // App3 should get 2 containers allocated (1 AM + 1 container). + Assert.assertEquals(2, schedulerApp3.getLiveContainers().size()); + + rm1.close(); + } + @Test public void testSchedulingRequestDisabledByDefault() throws Exception { Configuration csConf = TestUtils.getConfigurationWithMultipleQueues( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java index 76f451e919be0..cbf59682648ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; @@ -96,7 +97,9 @@ public void testAllocationTagsManagerSimpleCases() // Get Node Cardinality of app1 on node1, with tag "mapper" Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::max)); // Get Rack Cardinality of app1 on rack0, with tag "mapper" @@ -106,20 +109,26 @@ public void testAllocationTagsManagerSimpleCases() // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=min Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::min)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=max Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::max)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=sum Assert.assertEquals(3, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::sum)); // Get Node Cardinality by passing single tag. Assert.assertEquals(1, @@ -134,38 +143,52 @@ public void testAllocationTagsManagerSimpleCases() // op=min Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("no_existed", "reducer"), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("no_existed", "reducer")), + Long::min)); // Get Node Cardinality of app1 on node2, with tag "", op=max // (Expect this returns #containers from app1 on node2) Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), null, Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), null), + Long::max)); // Get Node Cardinality of app1 on node2, with empty tag set, op=max Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), null, Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), null), + Long::max)); // Get Cardinality of app1 on node2, with empty tag set, op=max Assert.assertEquals(2, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), ImmutableSet.of()), + Long::max)); // Get Node Cardinality of all apps on node2, with empty tag set, op=sum Assert.assertEquals(4, atm.getNodeCardinalityByOp( - NodeId.fromString("host2:123"), null, ImmutableSet.of(), Long::sum)); + NodeId.fromString("host2:123"), + AllocationTags.createGlobalAllocationTags(ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum Assert.assertEquals(3, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), ImmutableSet.of()), + Long::sum)); // Finish all containers: atm.removeContainer(NodeId.fromString("host1:123"), @@ -189,33 +212,42 @@ public void testAllocationTagsManagerSimpleCases() // Get Cardinality of app1 on node1, with tag "mapper" Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::max)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=min Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::min)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=max Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::max)); // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of("mapper", "reducer"), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper", "reducer")), + Long::sum)); // Get Node Cardinality of app1 on node2, with tag "", op=max // (Expect this returns #containers from app1 on node2) Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), - ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of(TestUtils.getMockApplicationId(1).toString())), Long::max)); Assert.assertEquals(0, @@ -226,21 +258,32 @@ public void testAllocationTagsManagerSimpleCases() // Get Node Cardinality of app1 on node2, with empty tag set, op=max Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::max)); // Get Node Cardinality of all apps on node2, with empty tag set, op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp( - NodeId.fromString("host2:123"), null, ImmutableSet.of(), Long::sum)); + NodeId.fromString("host2:123"), + AllocationTags.createGlobalAllocationTags(ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::sum)); // Get Node Cardinality of app_2 on node2, with empty tag set, op=sum Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::sum)); } @@ -296,20 +339,26 @@ public void testAllocationTagsManagerRackMapping() // Get Rack Cardinality of app_1 on rack0, with empty tag set, op=max Assert.assertEquals(1, atm.getRackCardinalityByOp("rack0", - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::max)); // Get Rack Cardinality of app_1 on rack0, with empty tag set, op=min Assert.assertEquals(1, atm.getRackCardinalityByOp("rack0", - TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::min)); + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of()), + Long::min)); // Get Rack Cardinality of all apps on rack0, with empty tag set, op=min - Assert.assertEquals(3, atm.getRackCardinalityByOp("rack0", null, - ImmutableSet.of(), Long::max)); + Assert.assertEquals(3, atm.getRackCardinalityByOp("rack0", + AllocationTags.createGlobalAllocationTags(ImmutableSet.of()), + Long::max)); } @Test - public void testAllocationTagsManagerMemoryAfterCleanup() - throws InvalidAllocationTagsQueryException { + public void testAllocationTagsManagerMemoryAfterCleanup() { /** * Make sure YARN cleans up all memory once container/app finishes. */ @@ -362,8 +411,7 @@ public void testAllocationTagsManagerMemoryAfterCleanup() } @Test - public void testQueryCardinalityWithIllegalParameters() - throws InvalidAllocationTagsQueryException { + public void testQueryCardinalityWithIllegalParameters() { /** * Make sure YARN cleans up all memory once container/app finishes. */ @@ -391,9 +439,12 @@ public void testQueryCardinalityWithIllegalParameters() // No node-id boolean caughtException = false; try { - atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2), - ImmutableSet.of("mapper"), Long::min); - } catch (InvalidAllocationTagsQueryException e) { + atm.getNodeCardinalityByOp(null, + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("mapper")), + Long::min); + } catch (InvalidAllocationTagsQueryException e1) { caughtException = true; } Assert.assertTrue("should fail because of nodeId specified", @@ -403,11 +454,150 @@ public void testQueryCardinalityWithIllegalParameters() caughtException = false; try { atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null); - } catch (InvalidAllocationTagsQueryException e) { + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("mapper")), + null); + } catch (InvalidAllocationTagsQueryException e1) { caughtException = true; } Assert.assertTrue("should fail because of nodeId specified", caughtException); } + + @Test + public void testNodeAllocationTagsAggregation() + throws InvalidAllocationTagsQueryException { + + AllocationTagsManager atm = new AllocationTagsManager(rmContext); + ApplicationId app1 = TestUtils.getMockApplicationId(1); + ApplicationId app2 = TestUtils.getMockApplicationId(2); + ApplicationId app3 = TestUtils.getMockApplicationId(3); + NodeId host1 = NodeId.fromString("host1:123"); + NodeId host2 = NodeId.fromString("host2:123"); + NodeId host3 = NodeId.fromString("host3:123"); + + /** + * Node1 (rack0) + * app1/A(2) + * app1/B(1) + * app2/A(3) + * app3/A(1) + * + * Node2 (rack0) + * app2/A(1) + * app2/B(2) + * app1/C(1) + * app3/B(1) + * + * Node3 (rack1): + * app2/D(1) + * app3/D(1) + */ + atm.addContainer(host1, TestUtils.getMockContainerId(1, 1), + ImmutableSet.of("A", "B")); + atm.addContainer(host1, TestUtils.getMockContainerId(1, 2), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(2, 1), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(2, 2), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(2, 3), + ImmutableSet.of("A")); + atm.addContainer(host1, TestUtils.getMockContainerId(3, 1), + ImmutableSet.of("A")); + + atm.addContainer(host2, TestUtils.getMockContainerId(1, 3), + ImmutableSet.of("C")); + atm.addContainer(host2, TestUtils.getMockContainerId(2, 4), + ImmutableSet.of("A")); + atm.addContainer(host2, TestUtils.getMockContainerId(2, 5), + ImmutableSet.of("B")); + atm.addContainer(host2, TestUtils.getMockContainerId(2, 6), + ImmutableSet.of("B")); + atm.addContainer(host2, TestUtils.getMockContainerId(3, 2), + ImmutableSet.of("B")); + + atm.addContainer(host3, TestUtils.getMockContainerId(2, 7), + ImmutableSet.of("D")); + atm.addContainer(host3, TestUtils.getMockContainerId(3, 3), + ImmutableSet.of("D")); + + // Target applications, current app: app1 + // all apps: app1, app2, app3 + TargetApplications ta = new TargetApplications(app1, + ImmutableSet.of(app1, app2, app3)); + + //******************************** + // 1) self (app1) + //******************************** + AllocationTags tags = AllocationTags + .createSingleAppAllocationTags(app1, ImmutableSet.of("A", "C")); + Assert.assertEquals(2, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host2, tags, Long::min)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::min)); + + //******************************** + // 2) not-self (app2, app3) + //******************************** + /** + * Verify max/min cardinality of tag "A" on host1 from all applications + * other than app1. This returns the max/min cardinality of tag "A" of + * app2 or app3 on this node. + * + * Node1 (rack0) + * app1/A(1) + * app1/B(1) + * app2/A(3) + * app3/A(1) + * + * app2_app3/A(4) + * app2_app3/B(0) + * + * expecting to return max=3, min=1 + * + */ + tags = AllocationTags.createOtherAppAllocationTags(app1, + ImmutableSet.of(app1, app2, app3), ImmutableSet.of("A", "B")); + + Assert.assertEquals(4, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(4, atm.getNodeCardinalityByOp(host1, tags, Long::sum)); + + //******************************** + // 3) app-id/app2 (app2) + //******************************** + tags = AllocationTags + .createSingleAppAllocationTags(app2, ImmutableSet.of("A", "B")); + Assert.assertEquals(3, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(2, atm.getNodeCardinalityByOp(host2, tags, Long::max)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::min)); + Assert.assertEquals(3, atm.getNodeCardinalityByOp(host2, tags, Long::sum)); + + + //******************************** + // 4) all (app1, app2, app3) + //******************************** + tags = AllocationTags + .createGlobalAllocationTags(ImmutableSet.of("A")); + Assert.assertEquals(6, atm.getNodeCardinalityByOp(host1, tags, Long::sum)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::sum)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::sum)); + + tags = AllocationTags + .createGlobalAllocationTags(ImmutableSet.of("A", "B")); + Assert.assertEquals(7, atm.getNodeCardinalityByOp(host1, tags, Long::sum)); + Assert.assertEquals(4, atm.getNodeCardinalityByOp(host2, tags, Long::sum)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::sum)); + Assert.assertEquals(6, atm.getNodeCardinalityByOp(host1, tags, Long::max)); + Assert.assertEquals(3, atm.getNodeCardinalityByOp(host2, tags, Long::max)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::max)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host1, tags, Long::min)); + Assert.assertEquals(1, atm.getNodeCardinalityByOp(host2, tags, Long::min)); + Assert.assertEquals(0, atm.getNodeCardinalityByOp(host3, tags, Long::min)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java index 67a3901e64401..d1ef331b19512 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsNamespace.java @@ -16,10 +16,8 @@ * limitations under the License. */ import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; +import org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.TargetApplications; -import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; import org.junit.Assert; import org.junit.Test; @@ -29,29 +27,34 @@ public class TestAllocationTagsNamespace { @Test - public void testNamespaceParse() throws InvalidAllocationTagException { + public void testNamespaceParse() throws InvalidAllocationTagsQueryException { AllocationTagNamespace namespace; String namespaceStr = "self"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isIntraApp()); + Assert.assertEquals(AllocationTagNamespaceType.SELF, + namespace.getNamespaceType()); namespaceStr = "not-self"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isNotSelf()); + Assert.assertEquals(AllocationTagNamespaceType.NOT_SELF, + namespace.getNamespaceType()); namespaceStr = "all"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isGlobal()); + Assert.assertEquals(AllocationTagNamespaceType.ALL, + namespace.getNamespaceType()); namespaceStr = "app-label"; namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isAppLabel()); + Assert.assertEquals(AllocationTagNamespaceType.APP_LABEL, + namespace.getNamespaceType()); ApplicationId applicationId = ApplicationId.newInstance(12345, 1); namespaceStr = "app-id/" + applicationId.toString(); namespace = AllocationTagNamespace.parse(namespaceStr); - Assert.assertTrue(namespace.isSingleInterApp()); + Assert.assertEquals(AllocationTagNamespaceType.APP_ID, + namespace.getNamespaceType()); // Invalid app-id namespace syntax, invalid app ID. try { @@ -59,7 +62,7 @@ public void testNamespaceParse() throws InvalidAllocationTagException { AllocationTagNamespace.parse(namespaceStr); Assert.fail("Parsing should fail as the given app ID is invalid"); } catch (Exception e) { - Assert.assertTrue(e instanceof InvalidAllocationTagException); + Assert.assertTrue(e instanceof InvalidAllocationTagsQueryException); Assert.assertTrue(e.getMessage().startsWith( "Invalid application ID for app-id")); } @@ -71,7 +74,7 @@ public void testNamespaceParse() throws InvalidAllocationTagException { Assert.fail("Parsing should fail as the given namespace" + " is missing application ID"); } catch (Exception e) { - Assert.assertTrue(e instanceof InvalidAllocationTagException); + Assert.assertTrue(e instanceof InvalidAllocationTagsQueryException); Assert.assertTrue(e.getMessage().startsWith( "Missing the application ID in the namespace string")); } @@ -82,14 +85,15 @@ public void testNamespaceParse() throws InvalidAllocationTagException { AllocationTagNamespace.parse(namespaceStr); Assert.fail("Parsing should fail as the giving type is not supported."); } catch (Exception e) { - Assert.assertTrue(e instanceof InvalidAllocationTagException); + Assert.assertTrue(e instanceof InvalidAllocationTagsQueryException); Assert.assertTrue(e.getMessage().startsWith( "Invalid namespace prefix")); } } @Test - public void testNamespaceEvaluation() throws InvalidAllocationTagException { + public void testNamespaceEvaluation() throws + InvalidAllocationTagsQueryException { AllocationTagNamespace namespace; TargetApplications targetApplications; ApplicationId app1 = ApplicationId.newInstance(10000, 1); @@ -131,10 +135,8 @@ public void testNamespaceEvaluation() throws InvalidAllocationTagException { namespaceStr = "all"; namespace = AllocationTagNamespace.parse(namespaceStr); - targetApplications = new TargetApplications(null, - ImmutableSet.of(app1, app2)); - namespace.evaluate(targetApplications); - Assert.assertEquals(2, namespace.getNamespaceScope().size()); + Assert.assertEquals(AllocationTagNamespaceType.ALL, + namespace.getNamespaceType()); namespaceStr = "app-id/" + app2.toString(); namespace = AllocationTagNamespace.parse(namespaceStr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java index 5ba89488af144..4814321932606 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java @@ -41,7 +41,6 @@ import java.util.stream.Stream; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.yarn.api.records.AllocationTagNamespace; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -512,6 +511,252 @@ public void testANDConstraintAssignment() createSchedulingRequest(sourceTag1), schedulerNode3, pcm, tm)); } + @Test + public void testGlobalAppConstraints() + throws InvalidAllocationTagsQueryException { + AllocationTagsManager tm = new AllocationTagsManager(rmContext); + PlacementConstraintManagerService pcm = + new MemoryPlacementConstraintManager(); + rmContext.setAllocationTagsManager(tm); + rmContext.setPlacementConstraintManager(pcm); + + long ts = System.currentTimeMillis(); + ApplicationId application1 = BuilderUtils.newApplicationId(ts, 100); + ApplicationId application2 = BuilderUtils.newApplicationId(ts, 101); + ApplicationId application3 = BuilderUtils.newApplicationId(ts, 102); + + // Register App1 with anti-affinity constraint map. + RMNode n0r1 = rmNodes.get(0); + RMNode n1r1 = rmNodes.get(1); + RMNode n2r2 = rmNodes.get(2); + RMNode n3r2 = rmNodes.get(3); + + /** + * Place container: + * n0: app1/A(1), app2/A(1) + * n1: app3/A(3) + * n2: app1/A(2) + * n3: "" + */ + tm.addContainer(n0r1.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n0r1.getNodeID(), + newContainerId(application2), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + + SchedulerNode schedulerNode0 = newSchedulerNode(n0r1.getHostName(), + n0r1.getRackName(), n0r1.getNodeID()); + SchedulerNode schedulerNode1 = newSchedulerNode(n1r1.getHostName(), + n1r1.getRackName(), n1r1.getNodeID()); + SchedulerNode schedulerNode2 = newSchedulerNode(n2r2.getHostName(), + n2r2.getRackName(), n2r2.getNodeID()); + SchedulerNode schedulerNode3 = newSchedulerNode(n3r2.getHostName(), + n3r2.getRackName(), n3r2.getNodeID()); + + AllocationTagNamespace namespaceAll = + new AllocationTagNamespace.All(); + + //*************************** + // 1) all, anti-affinity + //*************************** + // Anti-affinity with "A" from any application including itself. + PlacementConstraint constraint1 = PlacementConstraints.targetNotIn( + NODE, allocationTagWithNamespace(namespaceAll.toString(), "A")) + .build(); + Map, PlacementConstraint> constraintMap = new HashMap<>(); + Set srcTags1 = ImmutableSet.of("A"); + constraintMap.put(srcTags1, constraint1); + pcm.registerApplication(application1, constraintMap); + + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode0, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode1, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode2, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application1); + + //*************************** + // 2) all, max cardinality + //*************************** + PlacementConstraint constraint2 = PlacementConstraints + .maxCardinality(NODE, namespaceAll.toString(), 2, "A") + .build(); + constraintMap.clear(); + Set srcTags2 = ImmutableSet.of("foo"); + constraintMap.put(srcTags2, constraint2); + pcm.registerApplication(application2, constraintMap); + + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode0, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode1, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode2, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application2, createSchedulingRequest(srcTags2), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application2); + + //*************************** + // 3) all, min cardinality + //*************************** + PlacementConstraint constraint3 = PlacementConstraints + .minCardinality(NODE, namespaceAll.toString(), 3, "A") + .build(); + constraintMap.clear(); + Set srcTags3 = ImmutableSet.of("foo"); + constraintMap.put(srcTags3, constraint3); + pcm.registerApplication(application3, constraintMap); + + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode0, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode1, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode2, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application3, createSchedulingRequest(srcTags3), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application3); + } + + @Test + public void testNotSelfAppConstraints() + throws InvalidAllocationTagsQueryException { + AllocationTagsManager tm = new AllocationTagsManager(rmContext); + PlacementConstraintManagerService pcm = + new MemoryPlacementConstraintManager(); + rmContext.setAllocationTagsManager(tm); + rmContext.setPlacementConstraintManager(pcm); + + long ts = System.currentTimeMillis(); + ApplicationId application1 = BuilderUtils.newApplicationId(ts, 100); + ApplicationId application2 = BuilderUtils.newApplicationId(ts, 101); + ApplicationId application3 = BuilderUtils.newApplicationId(ts, 102); + + // Register App1 with anti-affinity constraint map. + RMNode n0r1 = rmNodes.get(0); + RMNode n1r1 = rmNodes.get(1); + RMNode n2r2 = rmNodes.get(2); + RMNode n3r2 = rmNodes.get(3); + + /** + * Place container: + * n0: app1/A(1), app2/A(1) + * n1: app3/A(3) + * n2: app1/A(2) + * n3: "" + */ + tm.addContainer(n0r1.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n0r1.getNodeID(), + newContainerId(application2), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n1r1.getNodeID(), + newContainerId(application3), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + tm.addContainer(n2r2.getNodeID(), + newContainerId(application1), ImmutableSet.of("A")); + + SchedulerNode schedulerNode0 = newSchedulerNode(n0r1.getHostName(), + n0r1.getRackName(), n0r1.getNodeID()); + SchedulerNode schedulerNode1 = newSchedulerNode(n1r1.getHostName(), + n1r1.getRackName(), n1r1.getNodeID()); + SchedulerNode schedulerNode2 = newSchedulerNode(n2r2.getHostName(), + n2r2.getRackName(), n2r2.getNodeID()); + SchedulerNode schedulerNode3 = newSchedulerNode(n3r2.getHostName(), + n3r2.getRackName(), n3r2.getNodeID()); + + AllocationTagNamespace notSelf = + new AllocationTagNamespace.NotSelf(); + + //*************************** + // 1) not-self, app1 + //*************************** + // Anti-affinity with "A" from app2 and app3, + // n0 and n1 both have tag "A" from either app2 or app3, so they are + // not qualified for the placement. + PlacementConstraint constraint1 = PlacementConstraints.targetNotIn( + NODE, allocationTagWithNamespace(notSelf.toString(), "A")) + .build(); + Map, PlacementConstraint> constraintMap = new HashMap<>(); + Set srcTags1 = ImmutableSet.of("A"); + constraintMap.put(srcTags1, constraint1); + pcm.registerApplication(application1, constraintMap); + + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode0, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode1, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode2, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags1), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application1); + + //*************************** + // 2) not-self, app1 + //*************************** + // Affinity with "A" from app2 and app3, + // N0 and n1 are qualified for the placement. + PlacementConstraint constraint2 = PlacementConstraints.targetIn( + NODE, allocationTagWithNamespace(notSelf.toString(), "A")) + .build(); + Map, PlacementConstraint> cm2 = new HashMap<>(); + Set srcTags2 = ImmutableSet.of("A"); + cm2.put(srcTags2, constraint2); + pcm.registerApplication(application1, cm2); + + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode0, pcm, tm)); + Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode1, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode2, pcm, tm)); + Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints( + application1, createSchedulingRequest(srcTags2), + schedulerNode3, pcm, tm)); + + pcm.unregisterApplication(application1); + } + @Test public void testInterAppConstraintsByAppID() throws InvalidAllocationTagsQueryException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java index 0b9657f15dedf..2ac5c3dc28c40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/TestLocalAllocationTagsManager.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException; import org.junit.Assert; @@ -85,46 +86,62 @@ public void testTempContainerAllocations() // Expect tag mappings to be present including temp Tags Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::sum)); Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("service")), Long::sum)); Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("service")), Long::sum)); // Do a temp Tag cleanup on app2 ephAtm.cleanTempContainers(TestUtils.getMockApplicationId(2)); Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("service")), Long::sum)); // Expect app1 to be unaffected Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::sum)); // Do a cleanup on app1 as well ephAtm.cleanTempContainers(TestUtils.getMockApplicationId(1)); Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("mapper")), Long::sum)); // Non temp-tags should be unaffected Assert.assertEquals(1, atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"), - TestUtils.getMockApplicationId(1), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(1), + ImmutableSet.of("service")), Long::sum)); Assert.assertEquals(0, atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"), - TestUtils.getMockApplicationId(2), ImmutableSet.of("service"), + AllocationTags.createSingleAppAllocationTags( + TestUtils.getMockApplicationId(2), + ImmutableSet.of("service")), Long::sum)); // Expect app2 with no containers, and app1 with 2 containers across 2 nodes diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java index 9be56ff0c4a4b..4c6afd4197646 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/TestSingleConstraintAppPlacementAllocator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.NodeId; @@ -366,8 +367,7 @@ public void testFunctionality() throws InvalidAllocationTagsQueryException { allocator.canAllocate(NodeType.NODE_LOCAL, TestUtils.getMockNode("host1", "/rack1", 123, 1024)); verify(spyAllocationTagsManager, Mockito.times(1)).getNodeCardinalityByOp( - eq(NodeId.fromString("host1:123")), eq(TestUtils.getMockApplicationId(1)), - eq(ImmutableSet.of("mapper", "reducer")), + eq(NodeId.fromString("host1:123")), any(AllocationTags.class), any(LongBinaryOperator.class)); allocator = new SingleConstraintAppPlacementAllocator(); @@ -388,9 +388,8 @@ public void testFunctionality() throws InvalidAllocationTagsQueryException { allocator.canAllocate(NodeType.NODE_LOCAL, TestUtils.getMockNode("host1", "/rack1", 123, 1024)); verify(spyAllocationTagsManager, Mockito.atLeast(1)).getNodeCardinalityByOp( - eq(NodeId.fromString("host1:123")), - eq(TestUtils.getMockApplicationId(1)), eq(ImmutableSet - .of("mapper", "reducer")), any(LongBinaryOperator.class)); + eq(NodeId.fromString("host1:123")), any(AllocationTags.class), + any(LongBinaryOperator.class)); SchedulerNode node1 = mock(SchedulerNode.class); when(node1.getPartition()).thenReturn("x"); From 1a125dd92f9a63e641a2266ccc871ada0a7f3000 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 4 Apr 2018 09:22:35 +0530 Subject: [PATCH 049/512] YARN-7764. Findbugs warning: Resource#getResources may expose internal representation. Contributed by Weiwei Yang. (cherry picked from commit f7a17b029ddd61ca73c2c2c88f5451dbf05fc501) --- .../hadoop-yarn/dev-support/findbugs-exclude.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 81b8825656042..58413618df037 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -658,4 +658,11 @@ + + + + + + + From 2f326159f14814ce669157d835597e4ed4555d37 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 3 Apr 2018 21:06:24 -0700 Subject: [PATCH 050/512] YARN-8106. Update LogAggregationIndexedFileController to use readFull instead read to avoid IOException while loading log meta. (Prabhu Joseph via wangda) Change-Id: I63a65f73f8d1636e2c99ed9c8c2bbd05efcff80f (cherry picked from commit b779f4f0f614fe47e05bc2be5494cf3cbcf6f63c) --- .../ifile/LogAggregationIndexedFileController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index 5bba2e0a409a9..a8ae06f876c84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -865,7 +865,8 @@ public IndexedLogsMeta loadIndexedLogsMeta(Path remoteLogPath, long end, byte[] array = new byte[offset]; fsDataIStream.seek( fileLength - offset - Integer.SIZE/ Byte.SIZE - UUID_LENGTH); - int actual = fsDataIStream.read(array); + fsDataIStream.readFully(array); + int actual = array.length; if (actual != offset) { throw new IOException("Error on loading log meta from " + remoteLogPath); From 4db13cb9442afd981f32365369bf9eec9eeca9a7 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 4 Apr 2018 22:13:14 +0530 Subject: [PATCH 051/512] YARN-8115. [UI2] URL data like nodeHTTPAddress must be encoded in UI before using to access NM. Contributed by Sreenath Somarajapuram. (cherry picked from commit 42cd367c9308b944bc71de6c07b6c3f028a0d874) --- .../webapp/app/components/node-menu-panel.js | 25 +++++++++++++++++++ .../webapp/app/controllers/yarn-node-app.js | 3 ++- .../webapp/app/controllers/yarn-node-apps.js | 3 ++- .../app/controllers/yarn-node-container.js | 3 ++- .../app/controllers/yarn-node-containers.js | 3 ++- .../main/webapp/app/controllers/yarn-node.js | 3 ++- .../app/controllers/yarn-nodes/table.js | 2 +- .../src/main/webapp/app/helpers/node-link.js | 2 +- .../main/webapp/app/initializers/loader.js | 1 + .../main/webapp/app/routes/yarn-node-apps.js | 8 +++--- .../webapp/app/routes/yarn-node-containers.js | 8 +++--- .../src/main/webapp/app/routes/yarn-node.js | 8 +++--- .../templates/components/node-menu-panel.hbs | 8 +++--- 13 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/node-menu-panel.js diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/node-menu-panel.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/node-menu-panel.js new file mode 100644 index 0000000000000..31457bed20a96 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/node-menu-panel.js @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ember from 'ember'; + +export default Ember.Component.extend({ + encodedAddr : Ember.computed("nodeAddr", function(){ + return encodeURIComponent(this.get('nodeAddr')); + }) +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js index 3dc09fcf7e80a..e0d58ecc41012 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js @@ -22,6 +22,7 @@ export default Ember.Controller.extend({ breadcrumbs: Ember.computed('model.nodeInfo', function () { var nodeInfo = this.get('model.nodeInfo'); + var addr = encodeURIComponent(nodeInfo.addr); return [{ text: "Home", routeName: 'application' @@ -30,7 +31,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-nodes.table' }, { text: `Node [ ${nodeInfo.id} ]`, - href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`, + href: `#/yarn-node/${nodeInfo.id}/${addr}/info`, }, { text: `Application [ ${nodeInfo.appId} ]`, }]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js index 6f53e74f287dd..ddc8256a73cac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js @@ -22,6 +22,7 @@ export default Ember.Controller.extend({ breadcrumbs: Ember.computed("model.attempt.appId", function () { var nodeInfo = this.get("model.nodeInfo"); + var addr = encodeURIComponent(nodeInfo.addr); return [{ text: "Home", routeName: 'application' @@ -30,7 +31,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-nodes.table' }, { text: `Node [ ${nodeInfo.id} ]`, - href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}` + href: `#/yarn-node/${nodeInfo.id}/${addr}/info` }, { text: "Applications", }]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js index afcd518488918..b8e7e36b04889 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js @@ -22,6 +22,7 @@ export default Ember.Controller.extend({ breadcrumbs: Ember.computed("model.nodeInfo", function () { var nodeInfo = this.get("model.nodeInfo"); + var addr = encodeURIComponent(nodeInfo.addr); return [{ text: "Home", routeName: 'application' @@ -30,7 +31,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-nodes.table' }, { text: `Node [ ${nodeInfo.id} ]`, - href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}` + href: `#/yarn-node/${nodeInfo.id}/${addr}/info` }, { text: `Container [ ${nodeInfo.containerId} ]` }]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js index 21d50a34d71d5..9fb03c814be79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js @@ -22,6 +22,7 @@ export default Ember.Controller.extend({ breadcrumbs: Ember.computed("model.nodeInfo", function () { var nodeInfo = this.get("model.nodeInfo"); + var addr = encodeURIComponent(nodeInfo.addr); return [{ text: "Home", routeName: 'application' @@ -30,7 +31,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-nodes.table' }, { text: `Node [ ${nodeInfo.id} ]`, - href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}` + href: `#/yarn-node/${nodeInfo.id}/${addr}/info` }, { text: "Containers", }]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js index 335a33c1cf6e3..c6cf2e34940ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js @@ -22,6 +22,7 @@ export default Ember.Controller.extend({ breadcrumbs: Ember.computed("model.nodeInfo", function () { var nodeInfo = this.get("model.nodeInfo"); + var addr = encodeURIComponent(nodeInfo.addr); return [{ text: "Home", @@ -31,7 +32,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-nodes.table' }, { text: `Node [ ${nodeInfo.id} ]`, - href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`, + href: `#/yarn-node/${nodeInfo.id}/${addr}/info`, }]; }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js index f6a6ee6daca43..0693d1b358dc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js @@ -69,7 +69,7 @@ export default Ember.Controller.extend({ facetType: null, getCellContent: function(row) { var node_id = row.get("id"), - node_addr = row.get("nodeHTTPAddress"), + node_addr = encodeURIComponent(row.get("nodeHTTPAddress")), href = `#/yarn-node/${node_id}/${node_addr}/info`; switch(row.get("nodeState")) { case "SHUTDOWN": diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/node-link.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/node-link.js index d71ac775a0f74..4f412da8a830d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/node-link.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/node-link.js @@ -29,7 +29,7 @@ export default Ember.Helper.helper(function(params,hash) { if (nodeState === "SHUTDOWN" || nodeState === "LOST") { html = html + nodeHTTPAddress; } else { - html = html + '' + + html = html + '' + nodeHTTPAddress + ''; } html = html + ''; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js index 47fe33e63a07b..af55036b480f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js @@ -44,6 +44,7 @@ function updateConfigs(application) { if(!ENV.hosts.rmWebAddress) { ENV.hosts.rmWebAddress = rmhost; + ENV.hosts.protocolScheme = window.location.protocol; } else { rmhost = ENV.hosts.rmWebAddress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js index 3a9855887697d..a1d9f153a6fec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-apps.js @@ -23,10 +23,12 @@ import AbstractRoute from './abstract'; export default AbstractRoute.extend({ model(param) { // Get all apps running on a specific node. Node is contacted by using node_addr. + var address = decodeURIComponent(param.node_addr); + address = address.replace(/(^\w+:|^)\/\//, ''); return Ember.RSVP.hash({ - apps: this.store.query('yarn-node-app', { nodeAddr: param.node_addr }), - nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}), - nodeInfo: { id: param.node_id, addr: param.node_addr } + apps: this.store.query('yarn-node-app', { nodeAddr: address }), + nmGpuInfo: this.store.findRecord('yarn-nm-gpu', address, {reload:true}), + nodeInfo: { id: param.node_id, addr: address } }); }, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js index 6968ce0915354..67c8fa64e6f44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-containers.js @@ -22,10 +22,12 @@ import AbstractRoute from './abstract'; export default AbstractRoute.extend({ model(param) { // Get all containers running on specific node. + var address = decodeURIComponent(param.node_addr); + address = address.replace(/(^\w+:|^)\/\//, ''); return Ember.RSVP.hash({ - containers: this.store.query('yarn-node-container', { nodeHttpAddr: param.node_addr }), - nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}), - nodeInfo: { id: param.node_id, addr: param.node_addr } + containers: this.store.query('yarn-node-container', { nodeHttpAddr: address }), + nmGpuInfo: this.store.findRecord('yarn-nm-gpu', address, {reload:true}), + nodeInfo: { id: param.node_id, addr: address } }); }, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js index 7ce615c83fe18..0e0b701acb443 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js @@ -23,10 +23,12 @@ import AbstractRoute from './abstract'; export default AbstractRoute.extend({ model(param) { // Fetches data from both NM and RM. RM is queried to get node usage info. + var address = decodeURIComponent(param.node_addr); + address = address.replace(/(^\w+:|^)\/\//, ''); return Ember.RSVP.hash({ - nodeInfo: { id: param.node_id, addr: param.node_addr }, - nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}), - node: this.store.findRecord('yarn-node', param.node_addr, {reload: true}), + nodeInfo: { id: param.node_id, addr: address }, + nmGpuInfo: this.store.findRecord('yarn-nm-gpu', address, {reload:true}), + node: this.store.findRecord('yarn-node', address, {reload: true}), rmNode: this.store.findRecord('yarn-rm-node', param.node_id, {reload: true}) }); }, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs index 966e408d2cfee..acdff2ffc9acf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs @@ -25,20 +25,20 @@