diff --git a/.travis.yml b/.travis.yml index b7c2bab257b..9bf17b446fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ matrix: env: PROFILE="-Phadoop-2.7" - jdk: "openjdk8" env: PROFILE="-Phadoop-cdh-2.6" + - jdk: "openjdk8" + env: PROFILE="-Phadoop-3.1" cache: directories: diff --git a/bin/common.sh b/bin/common.sh index 2dd43691fba..5138d110f1d 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -267,9 +267,13 @@ function smart_stop_daemon() { function reorder_lib() { local ajar="lib/jersey-core-1.9.jar" + local bjar="lib/jsr311-api-1.1.1.jar" if [ -f "${SMART_HOME}/${ajar}" ]; then SMART_CLASSPATH="${SMART_HOME}/${ajar}:${SMART_CLASSPATH}" fi + if [ -f "${SMART_HOME}/${bjar}" ]; then + SMART_CLASSPATH="${SMART_HOME}/${bjar}:${SMART_CLASSPATH}" + fi } function init_command() { diff --git a/docs/block-level-ec.md b/docs/block-level-ec.md index b33950a390f..230542bb6dd 100644 --- a/docs/block-level-ec.md +++ b/docs/block-level-ec.md @@ -14,7 +14,7 @@ lot from its design and experience. Striped EC vs Block EC ====================== -EC with striped layout can achieve storage saving with both small files and large files. It supports online erasure coding and when data writen to HDFS, the coding is already done. On high speed network environment, it outperforms than replica because at the same time it can write to more than one datanode in parallevel. While one drawback is it loses the data locality advantage which may impact the performance of upper layer applications especially for those particularly optimized according to the advantage. For more Striped EC introduction, +EC with striped layout can achieve storage saving with both small files and large files. It supports online erasure coding and when data writen to HDFS, the coding is already done. On high speed network environment, it outperforms than replica because at the same time it can write to more than one datanode in parallel. While one drawback is it loses the data locality advantage which may impact the performance of upper layer applications especially for those particularly optimized according to the advantage. For more Striped EC introduction, refer to this [our joint blog with Cloudera](https://blog.cloudera.com/blog/2015/09/introduction-to-hdfs-erasure-coding-in-apache-hadoop/). Block EC is very suitable for very large files of enough blocks needed by a erasure coding group. The erasure coding is done offline and in background instead of online while client writing data. Compared with striped EC, block EC keeps data locality, being of less performance impact to some frameworks and applications. diff --git a/docs/ssm-deployment-guide.md b/docs/ssm-deployment-guide.md index b7979e80476..049fabe9417 100755 --- a/docs/ssm-deployment-guide.md +++ b/docs/ssm-deployment-guide.md @@ -1,18 +1,18 @@ -# SSM Deployment Guide with Hadoop (CDH5.10.1 or Apache Hadoop 2.7.3) +# SSM Deployment Guide with Hadoop ---------------------------------------------------------------------------------- ## Requirements: -* Unix/Unix-like Operation System -* JDK 1.7 for CDH5.10.1 or JDK 1.8 for Apache Hadoop 2.7.3 -* CDH 5.10.1 or Apache Hadoop 2.7.3 +* Unix/Unix-like OS +* JDK 1.7 for CDH 5.10.1 or JDK 1.8 for Apache Hadoop 2.7.3/3.1.0 +* CDH 5.10.1 or Apache Hadoop 2.7.3/3.1.0 * MySQL Community 5.7.18+ -* Maven 3.1.1+ +* Maven 3.1.1+ (merely for build use) -## Why JDK 1.7 is preferred +## Why JDK 1.7 is preferred for CDH 5.10.1 - It is because by default CDH5.10.1 supports compile and run with JDK 1.7. If you + It is because by default CDH 5.10.1 supports compile and run with JDK 1.7. If you want to use JDK1.8, please turn to Cloudera web site for how to support JDK1.8 - in CDH5.10.1. + in CDH 5.10.1. For SSM, JDK 1.7 and 1.8 are both supported. @@ -24,7 +24,7 @@ Download SSM branch from Github https://github.com/Intel-bigdata/SSM/ ## **Build SSM** -### For CDH5.10.1 +### For CDH 5.10.1 `mvn clean package -Pdist,web,hadoop-cdh-2.6 -DskipTests` @@ -32,6 +32,9 @@ Download SSM branch from Github https://github.com/Intel-bigdata/SSM/ `mvn clean package -Pdist,web,hadoop-2.7 -DskipTests` +### For Hadoop 3.1.0 + + `mvn clean package -Pdist,web,hadoop-3.1 -DskipTests` A tar distribution package will be generated under 'smart-dist/target'. unzip the tar distribution package to ${SMART_HOME} directory, the configuration files of SSM is under '${SMART_HOME}/conf'. More detailed information, please refer to BUILDING.txt file. @@ -190,7 +193,9 @@ After finishing the SSM configuration, we can start to deploy the SSM package wi # Deploy SSM --------------------------------------------------------------------------------- -SSM supports two running modes, standalone service and SSM service with multiple Smart Agents. If file move performance is not the concern, then standalone service mode is enough. If better performance is desired, we recommend to deploy one agent on each Datanode. To deploy SSM to Smart Server nodes and Smart Agent nodes (if configured), you can enter into ${SMART_HOME} directory and type "./bin/install.sh". You can use --config to specify where SSM's config directory is. ${SMART_HOME}/conf is the default config directory if the config option is not used. +SSM supports two running modes, standalone service and SSM service with multiple Smart Agents. If file move performance is not the concern, then standalone service mode is enough. If better performance is desired, we recommend to deploy one agent on each Datanode. + +To deploy SSM to Smart Server nodes and Smart Agent nodes (if configured), please execute command `./bin/install.sh` in ${SMART_HOME}. You can use --config to specify where SSM's config directory is. ${SMART_HOME}/conf is the default config directory if the config option is not used. ## Standalone SSM Service @@ -251,12 +256,12 @@ Enter into ${SMART_HOME} directory for running SSM. You can type `./bin/ssm vers # Hadoop Configuration ---------------------------------------------------------------------------------- -After install CDH5.10.1 or Apache Hadoop 2.7.3, please do the following configurations for integrating SSM. +Please do the following configurations for integrating SSM with CDH 5.10.1, Apache Hadoop 2.7.3 or Apache Hadoop 3.1.0. **Warning: This step may lead to `Hadoop not working issue` if it's not configured correctly. So, during testing, we don't recommend changing any configurations in Hadoop.** Actually, SSM can work with an existing Hadoop cluster without any configuration change in Hadoop. Although in that case SSM cannot collect access count or data temperature from Hadoop, you can still use SSM action to change access count or data temperature. For example, you can use `read -file XXX` to change access count or data temperature of file `XXX`. -## Apache Hadoop 2.7.3 +## Apache Hadoop 2.7.3 or 3.1.0 ### core-site.xml changes @@ -327,7 +332,7 @@ Copy the SSM jars to the default Hadoop class path 2. Distribute the jars starts with smart to one of default hadoop classpath in each NameNode/DataNode. For example, copy SSM jars to `$HADOOP_HOME/share/hadoop/common/lib`. -## CDH5.10.1 +## CDH 5.10.1 ### core-site.xml changes @@ -527,7 +532,7 @@ SSM choose to save cmdlet and action execution history in metastore for audit an ``` SSM service restart is required after the configuration changes. -## Batch Size of Namespace fetcher +## Batch size of Namespace fetcher SSM will fetch/sync namespace from namenode when it is started. According to our tests, a large namespace may lead to long start up time. To avoid this, we add a parameter named `smart.namespace.fetcher.batch`, its default value is 500. You can change it if namespace is very large, e.g., 100M or more. A larger batch size will greatly speed up fetcher efficiency, and reduce start up time. ```xml @@ -551,7 +556,7 @@ SSM supports concurrently fetching namespace. You can set a large value for each Number of consumers in namespace fetcher ``` -## Disable SSM Client +## Disable SSM SmartDFSClient For some reasons, if you do want to disable SmartDFSClients on a specific host from contacting SSM server, it can be realized by using the following commands. After that, newly created SmartDFSClients on that node will not try to connect SSM server while other functions (like HDFS read/write) will remain unaffected. diff --git a/pom.xml b/pom.xml index 2e286b75f7d..04026a87f5f 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,22 @@ + + hadoop-3.1 + + smart-hadoop-support + + + 3.1.0 + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + alluxio @@ -277,6 +293,26 @@ com.google.code.findbugs jsr305 + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-webapp + + + org.eclipse.jetty.websocket + websocket-server + @@ -316,6 +352,18 @@ javax.servlet.jsp jsp-api + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-util-ajax + diff --git a/smart-agent/pom.xml b/smart-agent/pom.xml index 477619ac1c9..b2ee6584b8d 100644 --- a/smart-agent/pom.xml +++ b/smart-agent/pom.xml @@ -38,12 +38,6 @@ 1.5.0-SNAPSHOT provided - - org.smartdata - smart-hadoop-client - 1.5.0-SNAPSHOT - runtime - org.smartdata smart-tidb @@ -67,7 +61,7 @@ test - + diff --git a/smart-common/pom.xml b/smart-common/pom.xml index d80fdd5f31b..417fd71e4cd 100644 --- a/smart-common/pom.xml +++ b/smart-common/pom.xml @@ -39,6 +39,20 @@ org.apache.hadoop hadoop-common + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.eclipse.jetty + jetty-webapp + + junit diff --git a/smart-dist/pom.xml b/smart-dist/pom.xml index c090e47c38f..a412695e8ef 100644 --- a/smart-dist/pom.xml +++ b/smart-dist/pom.xml @@ -43,8 +43,22 @@ org.smartdata - smart-hadoop-client + smart-inputstream 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-2 + + + org.smartdata + smart-hadoop-2.7 + + + org.smartdata + smart-hadoop-client-2.7 + + org.smartdata @@ -62,10 +76,18 @@ 1.5.0-SNAPSHOT + + org.smartdata + smart-hadoop-2 + org.smartdata smart-hadoop-2.7 + + org.smartdata + smart-hadoop-client-2.7 + @@ -124,6 +146,11 @@ smart-hadoop-2.7 1.5.0-SNAPSHOT + + org.smartdata + smart-hadoop-client-2.7 + 1.5.0-SNAPSHOT + @@ -134,6 +161,26 @@ smart-hadoop-cdh-2.6 1.5.0-SNAPSHOT + + org.smartdata + smart-hadoop-client-cdh-2.6 + 1.5.0-SNAPSHOT + + + + + hadoop-3.1 + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-client-3.1 + 1.5.0-SNAPSHOT + diff --git a/smart-engine/pom.xml b/smart-engine/pom.xml index ec51c8f27ba..801a8724d09 100644 --- a/smart-engine/pom.xml +++ b/smart-engine/pom.xml @@ -46,11 +46,12 @@ org.smartdata smart-hadoop 1.5.0-SNAPSHOT - - - org.smartdata - smart-hadoop-client - 1.5.0-SNAPSHOT + + + commons-configuration + commons-configuration + + + smart-hadoop-2 smart-hadoop-2.7 + smart-hadoop-client-2.7 hadoop-cdh-2.6 + smart-hadoop-2 smart-hadoop-cdh-2.6 + smart-hadoop-client-cdh-2.6 + + + + hadoop-3.1 + + smart-hadoop-3.1 + smart-hadoop-client-3.1 diff --git a/smart-hadoop-support/smart-hadoop-2.7/pom.xml b/smart-hadoop-support/smart-hadoop-2.7/pom.xml index 250ae70f34a..0973f7db25f 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/pom.xml +++ b/smart-hadoop-support/smart-hadoop-2.7/pom.xml @@ -42,48 +42,24 @@ org.smartdata - smart-hadoop + smart-hadoop-common 1.5.0-SNAPSHOT org.smartdata - smart-hadoop + smart-hadoop-2 1.5.0-SNAPSHOT - test - test-jar - - - org.smartdata - smart-action - 1.5.0-SNAPSHOT - test - test-jar - - - org.apache.hadoop - hadoop-common - test - test-jar org.apache.hadoop hadoop-hdfs + ${hadoop.version} test test-jar - - junit - junit - test - - - org.mockito - mockito-all - test - org.smartdata - smart-metastore + smart-hadoop-common 1.5.0-SNAPSHOT test test-jar @@ -93,10 +69,5 @@ hadoop-aws ${hadoop.version} - - org.apache.httpcomponents - httpclient - 4.5.5 - diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/smartdata/hdfs/CompatibilityHelper27.java b/smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/smartdata/hdfs/CompatibilityHelper27.java index ae9419158ed..46fad6e8b75 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/smartdata/hdfs/CompatibilityHelper27.java +++ b/smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/smartdata/hdfs/CompatibilityHelper27.java @@ -22,8 +22,6 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.SmartInputStreamFactory; -import org.apache.hadoop.hdfs.SmartInputStreamFactory27; import org.apache.hadoop.hdfs.inotify.Event; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -45,7 +43,7 @@ import java.util.EnumSet; import java.util.List; -public class CompatibilityHelper27 implements CompatibilityHelper { +public class CompatibilityHelper27 extends CompatibilityHelper2 implements CompatibilityHelper { @Override public String[] getStorageTypes(LocatedBlock lb) { @@ -153,13 +151,20 @@ public int getSidInDatanodeStorageReport(DatanodeStorage datanodeStorage) { public OutputStream getDFSClientAppend(DFSClient client, String dest, int buffersize, long offset) throws IOException { if (client.exists(dest) && offset != 0) { - return client - .append(dest, buffersize, - EnumSet.of(CreateFlag.APPEND), null, null); + return getDFSClientAppend(client, dest, buffersize); } return client.create(dest, true); } + @Override + public OutputStream getDFSClientAppend( + DFSClient client, String dest, int buffersize) throws IOException { + return client + .append(dest, buffersize, + EnumSet.of(CreateFlag.APPEND), null, null); + } + + @Override public OutputStream getS3outputStream(String dest, Configuration conf) throws IOException { // Copy to remote S3 @@ -170,9 +175,4 @@ public OutputStream getS3outputStream(String dest, Configuration conf) throws IO org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(URI.create(dest), conf); return fs.create(new Path(dest), true); } - - @Override - public SmartInputStreamFactory getSmartInputStreamFactory() { - return new SmartInputStreamFactory27(); - } } diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestDataNodeInfoFetcher27.java b/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestDataNodeInfoFetcher27.java deleted file mode 100644 index 82ec84e1e69..00000000000 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestDataNodeInfoFetcher27.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.metrics.fetcher; - -import org.smartdata.hdfs.metric.fetcher.TestDataNodeInfoFetcher; - -public class TestDataNodeInfoFetcher27 extends TestDataNodeInfoFetcher{ -} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestEventBatchSerializer27.java b/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestEventBatchSerializer27.java deleted file mode 100644 index 839a58f60d5..00000000000 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestEventBatchSerializer27.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.metrics.fetcher; - -import org.smartdata.hdfs.metric.fetcher.TestEventBatchSerializer; - -public class TestEventBatchSerializer27 extends TestEventBatchSerializer { -} diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestInotifyFetcher27.java b/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestInotifyFetcher27.java deleted file mode 100644 index a9c45a32c22..00000000000 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestInotifyFetcher27.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.metrics.fetcher; - -import org.apache.hadoop.fs.CreateFlag; -import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; -import org.smartdata.hdfs.metric.fetcher.TestInotifyFetcher; - -import java.io.IOException; -import java.util.EnumSet; - -public class TestInotifyFetcher27 extends TestInotifyFetcher { - @Override - public HdfsDataOutputStream append(DFSClient client, String src, int bufferSize) throws IOException { - return client.append(src, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); - } -} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-2/pom.xml b/smart-hadoop-support/smart-hadoop-2/pom.xml new file mode 100644 index 00000000000..ebbb303b999 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-2/pom.xml @@ -0,0 +1,50 @@ + + + + 4.0.0 + + + org.smartdata + smart-hadoop-support + 1.5.0-SNAPSHOT + .. + + + smart-hadoop-2 + 1.5.0-SNAPSHOT + jar + + + + org.smartdata + smart-common + 1.5.0-SNAPSHOT + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.smartdata + smart-hadoop-common + 1.5.0-SNAPSHOT + + + diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java rename to smart-hadoop-support/smart-hadoop-2/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java similarity index 90% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java rename to smart-hadoop-support/smart-hadoop-2/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java index 7aa402a7cfd..a2b6310d846 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java +++ b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSInputStream; import org.smartdata.model.FileState; import java.io.IOException; @@ -25,7 +27,7 @@ /** * DFSInputStream for SSM. */ -public abstract class SmartInputStream extends DFSInputStream { +public class SmartInputStream extends DFSInputStream { protected final FileState fileState; SmartInputStream(DFSClient dfsClient, String src, boolean verifyChecksum, @@ -37,4 +39,4 @@ public abstract class SmartInputStream extends DFSInputStream { public FileState.FileType getType() { return fileState.getFileType(); } -} +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java new file mode 100644 index 00000000000..935321b452e --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.protocolPB.PBHelper; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.balancer.KeyManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.security.token.Token; +import org.smartdata.hdfs.action.move.StorageGroup; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class CompatibilityHelper2 { + public int getReadTimeOutConstant() { + return HdfsServerConstants.READ_TIMEOUT; + } + + public Token getAccessToken( + KeyManager km, ExtendedBlock eb, StorageGroup target) throws IOException { + return km.getAccessToken(eb); + } + + public int getIOFileBufferSize(Configuration conf) { + return HdfsConstants.IO_FILE_BUFFER_SIZE; + } + + public InputStream getVintPrefixed(DataInputStream in) throws IOException { + return PBHelper.vintPrefixed(in); + } + + public LocatedBlocks getLocatedBlocks(HdfsLocatedFileStatus status) { + return status.getBlockLocations(); + } + + public HdfsFileStatus createHdfsFileStatus( + long length, boolean isdir, int block_replication, long blocksize, long modification_time, + long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, + long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) { + return new HdfsFileStatus( + length, isdir, block_replication, blocksize, modification_time, access_time, permission, + owner, group, symlink, path, fileId, childrenNum, feInfo, storagePolicy); + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/pom.xml b/smart-hadoop-support/smart-hadoop-3.1/pom.xml new file mode 100644 index 00000000000..a68ac82c0ae --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/pom.xml @@ -0,0 +1,93 @@ + + + + 4.0.0 + + + org.smartdata + smart-hadoop-support + 1.5.0-SNAPSHOT + .. + + + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + jar + + + 3.1.0 + + + + + org.smartdata + smart-common + 1.5.0-SNAPSHOT + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-util-ajax + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + provided + + + org.smartdata + smart-hadoop-common + 1.5.0-SNAPSHOT + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + test + test-jar + + + org.smartdata + smart-hadoop-common + 1.5.0-SNAPSHOT + test + test-jar + + + org.apache.hadoop + hadoop-aws + ${hadoop.version} + + + diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java new file mode 100644 index 00000000000..3d192f54aa9 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/apache/hadoop/hdfs/CompactInputStream.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.fs.ReadOption; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.io.ByteBufferPool; +import org.smartdata.model.CompactFileState; +import org.smartdata.model.FileContainerInfo; +import org.smartdata.model.FileState; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +public class CompactInputStream extends SmartInputStream { + private FileContainerInfo fileContainerInfo; + private boolean closed = false; + private static final String INHERITED_CLASS = "org.apache.hadoop.hdfs.DFSInputStream"; + + CompactInputStream(DFSClient dfsClient, boolean verifyChecksum, + FileState fileState) throws IOException { + super(dfsClient, + ((CompactFileState) fileState).getFileContainerInfo().getContainerFilePath(), + verifyChecksum, + fileState); + this.fileContainerInfo = ((CompactFileState) fileState).getFileContainerInfo(); + super.seek(fileContainerInfo.getOffset()); + } + + @Override + public long getFileLength() { + String callerClass = Thread.currentThread().getStackTrace()[2].getClassName(); + if (INHERITED_CLASS.equals(callerClass)) { + return fileContainerInfo.getLength() + fileContainerInfo.getOffset(); + } else { + return fileContainerInfo.getLength(); + } + } + + @Override + public List getAllBlocks() throws IOException { + List blocks = super.getAllBlocks(); + List ret = new ArrayList<>(16); + long off = fileContainerInfo.getOffset(); + long len = fileContainerInfo.getLength(); + for (LocatedBlock b : blocks) { + if (off > b.getStartOffset() + b.getBlockSize() || off + len < b.getStartOffset()) { + continue; + } + ret.add(b); + } + return ret; + } + + @Override + public synchronized int read(final byte[] buf, int off, int len) throws IOException { + int realLen = (int) Math.min(len, fileContainerInfo.getLength() - getPos()); + return super.read(buf, off, realLen); + } + + @Override + public synchronized int read(final ByteBuffer buf) throws IOException { + int realLen = (int) Math.min(buf.remaining(), fileContainerInfo.getLength() - getPos()); + buf.limit(realLen + buf.position()); + return super.read(buf); + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) throws IOException { + int realLen = (int) Math.min(length, fileContainerInfo.getLength() - getPos()); + long realPos = position + fileContainerInfo.getOffset(); + return super.read(realPos, buffer, offset, realLen); + } + + @Override + public synchronized long getPos() { + String callerClass = Thread.currentThread().getStackTrace()[2].getClassName(); + if (INHERITED_CLASS.equals(callerClass)) { + return super.getPos(); + } else { + return super.getPos() - fileContainerInfo.getOffset(); + } + } + + @Override + public synchronized int available() throws IOException { + if (closed) { + throw new IOException("Stream closed."); + } + final long remaining = getFileLength() - getPos(); + return remaining <= Integer.MAX_VALUE ? (int) remaining : Integer.MAX_VALUE; + } + + @Override + public synchronized void seek(long targetPos) throws IOException { + String callerClass = Thread.currentThread().getStackTrace()[2].getClassName(); + if (INHERITED_CLASS.equals(callerClass)) { + super.seek(targetPos); + } else { + if (targetPos > fileContainerInfo.getLength()) { + throw new EOFException("Cannot seek after EOF"); + } + if (targetPos < 0) { + throw new EOFException("Cannot seek to negative offset"); + } + super.seek(fileContainerInfo.getOffset() + targetPos); + } + } + + @Override + public synchronized boolean seekToNewSource(long targetPos) throws IOException { + String callerClass = Thread.currentThread().getStackTrace()[2].getClassName(); + if (INHERITED_CLASS.equals(callerClass)) { + return super.seekToNewSource(targetPos); + } else { + if (targetPos < 0) { + throw new EOFException("Cannot seek after EOF"); + } else { + return super.seekToNewSource(fileContainerInfo.getOffset() + targetPos); + } + } + } + + @Override + public synchronized void setReadahead(Long readahead) throws IOException { + long realReadAhead = Math.min(readahead, fileContainerInfo.getLength() - getPos()); + super.setReadahead(realReadAhead); + } + + @Override + public synchronized ByteBuffer read(ByteBufferPool bufferPool, + int maxLength, EnumSet opts) + throws IOException, UnsupportedOperationException { + int realMaxLen = (int) Math.min(maxLength, fileContainerInfo.getLength() - getPos()); + return super.read(bufferPool, realMaxLen, opts); + } + + @Override + public synchronized void close() throws IOException { + super.close(); + this.closed = true; + } +} diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory26.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java similarity index 66% rename from smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory26.java rename to smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java index 8616a6863de..69f49aa30c8 100644 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory26.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/apache/hadoop/hdfs/SmartInputStream.java @@ -17,19 +17,23 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.fs.UnresolvedLinkException; import org.smartdata.model.FileState; import java.io.IOException; -public class SmartInputStreamFactory26 extends SmartInputStreamFactory { - @Override - public DFSInputStream create(DFSClient dfsClient, String src, - boolean verifyChecksum, FileState fileState) - throws IOException, UnresolvedLinkException { - dfsClient.checkOpen(); - DFSInputStream inputStream = createSmartInputStream(dfsClient, src, - verifyChecksum, fileState); - return inputStream; +/** + * DFSInputStream for SSM. + */ +public class SmartInputStream extends DFSInputStream { + protected final FileState fileState; + + SmartInputStream(DFSClient dfsClient, String src, boolean verifyChecksum, + FileState fileState) throws IOException { + super(dfsClient, src, verifyChecksum, dfsClient.getLocatedBlocks(src, 0)); + this.fileState = fileState; + } + + public FileState.FileType getType() { + return fileState.getFileType(); } } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java new file mode 100644 index 00000000000..a1ce43f0205 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.inotify.Event; +import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; +import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos; +import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.balancer.KeyManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.security.token.Token; +import org.smartdata.hdfs.action.move.StorageGroup; + +import java.io.*; +import java.net.URI; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +public class CompatibilityHelper31 implements CompatibilityHelper { + @Override + public String[] getStorageTypes(LocatedBlock lb) { + List types = new ArrayList<>(); + for(StorageType type : lb.getStorageTypes()) { + types.add(type.toString()); + } + return types.toArray(new String[types.size()]); + } + + @Override + public void replaceBlock( + DataOutputStream out, + ExtendedBlock eb, + String storageType, + Token accessToken, + String dnUUID, + DatanodeInfo info) + throws IOException { + new Sender(out).replaceBlock(eb, StorageType.valueOf(storageType), accessToken, dnUUID, info, null); + } + + @Override + public String[] getMovableTypes() { + List types = new ArrayList<>(); + for(StorageType type : StorageType.getMovableTypes()) { + types.add(type.toString()); + } + return types.toArray(new String[types.size()]); + } + + @Override + public String getStorageType(StorageReport report) { + return report.getStorage().getStorageType().toString(); + } + + @Override + public List chooseStorageTypes(BlockStoragePolicy policy, short replication) { + List types = new ArrayList<>(); + for(StorageType type : policy.chooseStorageTypes(replication)) { + types.add(type.toString()); + } + return types; + } + + @Override + public boolean isMovable(String type) { + return StorageType.valueOf(type).isMovable(); + } + + @Override + public DatanodeInfo newDatanodeInfo(String ipAddress, int xferPort) { + DatanodeID datanodeID = new DatanodeID(ipAddress, null, null, + xferPort, 0, 0, 0); + DatanodeDescriptor datanodeDescriptor = new DatanodeDescriptor(datanodeID); + return datanodeDescriptor; + } + + @Override + public InotifyProtos.AppendEventProto getAppendEventProto(Event.AppendEvent event) { + return InotifyProtos.AppendEventProto.newBuilder() + .setPath(event.getPath()) + .setNewBlock(event.toNewBlock()).build(); + } + + @Override + public Event.AppendEvent getAppendEvent(InotifyProtos.AppendEventProto proto) { + return new Event.AppendEvent.Builder().path(proto.getPath()) + .newBlock(proto.hasNewBlock() && proto.getNewBlock()) + .build(); + } + + @Override + public boolean truncate(DFSClient client, String src, long newLength) throws IOException { + return client.truncate(src, newLength); + } + + @Override + public boolean truncate(DistributedFileSystem fileSystem, String src, long newLength) throws IOException { + return fileSystem.truncate(new Path(src), newLength); + } + + @Override + public int getSidInDatanodeStorageReport(DatanodeStorage datanodeStorage) { + StorageType storageType = datanodeStorage.getStorageType(); + return storageType.ordinal(); + } + + @Override + public OutputStream getDFSClientAppend(DFSClient client, String dest, + int buffersize, long offset) throws IOException { + if (client.exists(dest) && offset != 0) { + return getDFSClientAppend(client, dest, buffersize); + } + return client.create(dest, true); + } + + @Override + public OutputStream getDFSClientAppend(DFSClient client, String dest, + int buffersize) throws IOException { + return client + .append(dest, buffersize, + EnumSet.of(CreateFlag.APPEND), null, null); + } + + @Override + public OutputStream getS3outputStream(String dest, Configuration conf) throws IOException { + // Copy to remote S3 + if (!dest.startsWith("s3")) { + throw new IOException(); + } + // Copy to s3 + org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(URI.create(dest), conf); + return fs.create(new Path(dest), true); + } + + @Override + public int getReadTimeOutConstant() { + return HdfsConstants.READ_TIMEOUT; + } + + @Override + public Token getAccessToken( + KeyManager km, ExtendedBlock eb, StorageGroup target) throws IOException { + return km.getAccessToken(eb, new StorageType[]{StorageType.parseStorageType(target.getStorageType())}, new String[0]); + } + + @Override + public int getIOFileBufferSize(Configuration conf) { + return conf.getInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, + CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT); + } + + @Override + public InputStream getVintPrefixed(DataInputStream in) throws IOException { + return PBHelperClient.vintPrefixed(in); + } + + @Override + public LocatedBlocks getLocatedBlocks(HdfsLocatedFileStatus status) { + return status.getLocatedBlocks(); + } + + @Override + public HdfsFileStatus createHdfsFileStatus( + long length, boolean isdir, int block_replication, long blocksize, long modification_time, + long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, + long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) { + return new HdfsFileStatus.Builder() + .length(length) + .isdir(isdir) + .replication(block_replication) + .blocksize(blocksize) + .mtime(modification_time) + .atime(access_time) + .perm(permission) + .owner(owner) + .group(group) + .symlink(symlink) + .path(path) + .fileId(fileId) + .children(childrenNum) + .feInfo(feInfo) + .storagePolicy(storagePolicy) + .build(); + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory27.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/MiniClusterFactory31.java similarity index 53% rename from smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory27.java rename to smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/MiniClusterFactory31.java index e920713b5bb..60b6771073b 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory27.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/MiniClusterFactory31.java @@ -15,27 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs; +package org.smartdata.hdfs; -import org.apache.htrace.TraceScope; -import org.smartdata.model.FileState; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.MiniDFSCluster; import java.io.IOException; -/** - * Factory to create SmartInputStream for Apache Hadoop 2.7. - */ -public class SmartInputStreamFactory27 extends SmartInputStreamFactory { +public class MiniClusterFactory31 extends MiniClusterFactory { + @Override + public MiniDFSCluster create(int dataNodes, Configuration conf) throws IOException { + return new MiniDFSCluster.Builder(conf) + .numDataNodes(dataNodes) + .build(); + } + @Override - public DFSInputStream create(DFSClient dfsClient, String src, - boolean verifyChecksum, FileState fileState) - throws IOException { - dfsClient.checkOpen(); - TraceScope scope = dfsClient.getPathTraceScope("newDFSInputStream", src); - try { - return createSmartInputStream(dfsClient, src, verifyChecksum, fileState); - } finally { - scope.close(); - } + public MiniDFSCluster createWithStorages(int dataNodes, Configuration conf) throws IOException { + return new MiniDFSCluster.Builder(conf) + .numDataNodes(dataNodes) + .storagesPerDatanode(3) + .storageTypes(new StorageType[]{StorageType.DISK, StorageType.ARCHIVE, + StorageType.SSD}) + .build(); } } diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/pom.xml b/smart-hadoop-support/smart-hadoop-cdh-2.6/pom.xml index 9256199bcb3..66d154387c6 100644 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/pom.xml +++ b/smart-hadoop-support/smart-hadoop-cdh-2.6/pom.xml @@ -35,65 +35,31 @@ - - org.smartdata - smart-hadoop - 1.5.0-SNAPSHOT - org.apache.hadoop hadoop-hdfs ${hadoop.version} - - junit - junit - test - - - org.mockito - mockito-all - test - org.smartdata - smart-hadoop + smart-hadoop-2 1.5.0-SNAPSHOT - test - test-jar - - - org.smartdata - smart-action - 1.5.0-SNAPSHOT - test - test-jar org.apache.hadoop - hadoop-common - test - test-jar + hadoop-hdfs + ${hadoop.version} org.apache.hadoop hadoop-hdfs + ${hadoop.version} test test-jar - - junit - junit - test - - - org.mockito - mockito-all - test - org.smartdata - smart-metastore + smart-hadoop-common 1.5.0-SNAPSHOT test test-jar diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/smartdata/hdfs/CompatibilityHelper26.java b/smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/smartdata/hdfs/CompatibilityHelper26.java index c6fc187d680..fd02c92d9f5 100644 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/smartdata/hdfs/CompatibilityHelper26.java +++ b/smart-hadoop-support/smart-hadoop-cdh-2.6/src/main/java/org/smartdata/hdfs/CompatibilityHelper26.java @@ -21,8 +21,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.SmartInputStreamFactory; -import org.apache.hadoop.hdfs.SmartInputStreamFactory26; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.inotify.Event; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; @@ -43,7 +41,7 @@ import java.util.ArrayList; import java.util.List; -public class CompatibilityHelper26 implements CompatibilityHelper { +public class CompatibilityHelper26 extends CompatibilityHelper2 implements CompatibilityHelper { @Override public String[] getStorageTypes(LocatedBlock lb) { List types = new ArrayList<>(); @@ -139,12 +137,18 @@ public int getSidInDatanodeStorageReport(DatanodeStorage datanodeStorage) { public OutputStream getDFSClientAppend(DFSClient client, String dest, int buffersize, long offset) throws IOException { if (client.exists(dest) && offset != 0) { - return client - .append(dest, buffersize, null, null); + return getDFSClientAppend(client, dest, buffersize); } return client.create(dest, true); } + @Override + public OutputStream getDFSClientAppend( + DFSClient client, String dest, int buffersize) throws IOException { + return client + .append(dest, buffersize, null, null); + } + @Override public OutputStream getS3outputStream(String dest, Configuration conf) throws IOException { // Copy to remote S3 @@ -155,9 +159,4 @@ public OutputStream getS3outputStream(String dest, Configuration conf) throws IO org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(URI.create(dest), conf); return fs.create(new Path(dest), true); } - - @Override - public SmartInputStreamFactory getSmartInputStreamFactory() { - return new SmartInputStreamFactory26(); - } } diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/action/TestCopy2S3Action.java b/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/action/TestCopy2S3Action.java deleted file mode 100644 index eebf4e3eb31..00000000000 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/action/TestCopy2S3Action.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.action; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.smartdata.SmartContext; -import org.smartdata.action.MockActionStatusReporter; -import org.smartdata.conf.SmartConf; -import org.smartdata.hdfs.MiniClusterHarness; - -import java.io.OutputStream; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; - -/** - * Test for Copy2S3FileAction. - */ -public class TestCopy2S3Action extends MiniClusterHarness { - // //change this to connect with aws s3 - // final String S3_ACCESS_KEY = ""; - // final String S3_SECRET_KEY = ""; - // - // private SmartContext testContext; - // - // @Before - // public void testinit(){ - // SmartConf configuration = smartContext.getConf(); - // testContext = smartContext; - // configuration.set("fs.s3a.access.key", S3_ACCESS_KEY); - // configuration.set("fs.s3a.secret.key", S3_SECRET_KEY); - // testContext.setConf(configuration); - // } - // - // private void copy2S3File(String src, String dest) throws Exception { - // Copy2S3Action copy2S3Action = new Copy2S3Action(); - // copy2S3Action.setDfsClient(dfsClient); - // copy2S3Action.setContext(testContext); - // copy2S3Action.setStatusReporter(new MockActionStatusReporter()); - // Map args = new HashMap<>(); - // args.put(Copy2S3Action.FILE_PATH, src); - // - // args.put(Copy2S3Action.DEST_PATH, dest); - // copy2S3Action.init(args); - // copy2S3Action.run(); - // } - // - // @Test - // public void testS3FileCopy() throws Exception { - // final String srcPath = "/testCopy"; - // final String file = "testFile"; - // - // String destFile = "s3a://xxxctest/"; - // - // //get unigue name with the help of system time - // destFile = destFile + System.currentTimeMillis(); - // dfs.mkdirs(new Path(srcPath)); - // // write to DISK - // final OutputStream out1 = dfsClient.create(srcPath + "/" + file, true); - // out1.write(1); - // out1.close(); - // copy2S3File(srcPath + "/" + file, destFile); - // - // //check the file in S3 - // - // FileSystem fs = FileSystem.get(URI.create(destFile), testContext.getConf()); - // - // Assert.assertTrue(fs.exists(new Path(destFile))); - // } - - -} diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestDataNodeInfoFetcher26.java b/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestDataNodeInfoFetcher26.java deleted file mode 100644 index 4e597f1dfb8..00000000000 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestDataNodeInfoFetcher26.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.metrics.fetcher; - -import org.smartdata.hdfs.metric.fetcher.TestDataNodeInfoFetcher; - -public class TestDataNodeInfoFetcher26 extends TestDataNodeInfoFetcher { -} diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestEventBatchSerializer26.java b/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestEventBatchSerializer26.java deleted file mode 100644 index c38a45d29de..00000000000 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestEventBatchSerializer26.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.metrics.fetcher; - -import org.smartdata.hdfs.metric.fetcher.TestEventBatchSerializer; - -public class TestEventBatchSerializer26 extends TestEventBatchSerializer { -} diff --git a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestInotifyFetcher26.java b/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestInotifyFetcher26.java deleted file mode 100644 index 45cadaa279e..00000000000 --- a/smart-hadoop-support/smart-hadoop-cdh-2.6/src/test/java/org/smartdata/hdfs/metrics/fetcher/TestInotifyFetcher26.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.metrics.fetcher; - -import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; -import org.smartdata.hdfs.metric.fetcher.TestInotifyFetcher; - -import java.io.IOException; - -public class TestInotifyFetcher26 extends TestInotifyFetcher { - @Override - public HdfsDataOutputStream append(DFSClient client, String src, int bufferSize) throws IOException { - return client.append(src, bufferSize, null, null); - } -} diff --git a/smart-hadoop-support/smart-hadoop-client/pom.xml b/smart-hadoop-support/smart-hadoop-client-2.7/pom.xml similarity index 90% rename from smart-hadoop-support/smart-hadoop-client/pom.xml rename to smart-hadoop-support/smart-hadoop-client-2.7/pom.xml index ae289d8f906..b2dba1c5695 100644 --- a/smart-hadoop-support/smart-hadoop-client/pom.xml +++ b/smart-hadoop-support/smart-hadoop-client-2.7/pom.xml @@ -26,7 +26,7 @@ .. - smart-hadoop-client + smart-hadoop-client-2.7 1.5.0-SNAPSHOT jar @@ -46,7 +46,7 @@ org.apache.hadoop hadoop-hdfs - 2.7.3 + ${hadoop.version} provided @@ -60,6 +60,12 @@ 1.5.0-SNAPSHOT provided + + org.smartdata + smart-inputstream + 1.5.0-SNAPSHOT + provided + diff --git a/smart-hadoop-support/smart-hadoop-client/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java b/smart-hadoop-support/smart-hadoop-client-2.7/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java similarity index 98% rename from smart-hadoop-support/smart-hadoop-client/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java rename to smart-hadoop-support/smart-hadoop-client-2.7/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java index e1bae82996d..bb05290f16e 100644 --- a/smart-hadoop-support/smart-hadoop-client/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java +++ b/smart-hadoop-support/smart-hadoop-client-2.7/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java @@ -124,16 +124,7 @@ public void setVerifyChecksum(boolean verifyChecksum) { @Override public FSDataOutputStream append(Path f, final int bufferSize, final Progressable progress) throws IOException { - try { - return super.append(f, bufferSize, progress); - } catch (IOException e) { - FileState fileState = smartDFSClient.getFileState(getPathName(f)); - if (fileState instanceof CompactFileState) { - throw new IOException( - smartDFSClient.getExceptionMsg("Append", "SSM Small File")); - } - throw e; - } + return append(f, EnumSet.of(CreateFlag.APPEND), bufferSize, progress); } @Override diff --git a/smart-hadoop-support/smart-hadoop-client/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java b/smart-hadoop-support/smart-hadoop-client-2.7/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java similarity index 98% rename from smart-hadoop-support/smart-hadoop-client/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java rename to smart-hadoop-support/smart-hadoop-client-2.7/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java index c01dddd81dc..62043601ff4 100644 --- a/smart-hadoop-support/smart-hadoop-client/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java +++ b/smart-hadoop-support/smart-hadoop-client-2.7/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java @@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory; import org.smartdata.SmartConstants; import org.smartdata.client.SmartClient; +import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.metrics.FileAccessEvent; import org.smartdata.model.CompactFileState; import org.smartdata.model.FileState; @@ -149,7 +150,7 @@ public DFSInputStream open(String src, int buffersize, throw new IOException("Cannot open " + src + " when it is under PROCESSING to " + fileState.getFileType()); } - is = SmartInputStreamFactory.get().create(this, src, + is = SmartInputStreamFactory.create(this, src, verifyChecksum, fileState); } reportFileAccessEvent(src); @@ -203,7 +204,7 @@ public HdfsFileStatus getFileInfo(String src) throws IOException { FileState fileState = getFileState(src); if (fileState instanceof CompactFileState) { long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); - return new HdfsFileStatus(len, oldStatus.isDir(), oldStatus.getReplication(), + return CompatibilityHelperLoader.getHelper().createHdfsFileStatus(len, oldStatus.isDir(), oldStatus.getReplication(), oldStatus.getBlockSize(), oldStatus.getModificationTime(), oldStatus.getAccessTime(), oldStatus.getPermission(), oldStatus.getOwner(), oldStatus.getGroup(), oldStatus.isSymlink() ? oldStatus.getSymlinkInBytes() : null, diff --git a/smart-hadoop-support/smart-hadoop-client-3.1/pom.xml b/smart-hadoop-support/smart-hadoop-client-3.1/pom.xml new file mode 100644 index 00000000000..f98fb09e3d7 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-client-3.1/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + + org.smartdata + smart-hadoop-support + 1.5.0-SNAPSHOT + .. + + + smart-hadoop-client-3.1 + 1.5.0-SNAPSHOT + jar + + + + org.smartdata + smart-common + 1.5.0-SNAPSHOT + provided + + + org.smartdata + smart-metrics + 1.5.0-SNAPSHOT + provided + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + provided + + + org.smartdata + smart-client + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop + 1.5.0-SNAPSHOT + provided + + + org.smartdata + smart-inputstream + 1.5.0-SNAPSHOT + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + package + + copy-dependencies + + + target/lib + + + + + + + diff --git a/smart-hadoop-support/smart-hadoop-client-3.1/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java b/smart-hadoop-support/smart-hadoop-client-3.1/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java new file mode 100644 index 00000000000..bb05290f16e --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-client-3.1/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java @@ -0,0 +1,697 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hadoop.filesystem; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemLinkResolver; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; +import org.apache.hadoop.util.Progressable; +import org.smartdata.conf.SmartConfKeys; +import org.smartdata.hdfs.client.SmartDFSClient; +import org.smartdata.model.CompactFileState; +import org.smartdata.model.FileContainerInfo; +import org.smartdata.model.FileState; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +/** + * SmartFileSystem Deploy Guide + * 1. Build SSM, get all jar files start with name Smart* + * 2. Copy these jar files to HDFS classpath + * 3. Reconfigure HDFS + * Please do the following configurations, + * 1. core-site.xml + * Change property "fs.hdfs.impl" value, to point to the Smart Server provided + * "Smart File System". + * + * fs.hdfs.impl + * org.smartdata.hadoop.filesystem.SmartFileSystem + * The FileSystem for hdfs URL + * + * 2. hdfs-site.xml + * Add property "smart.server.rpc.adddress" to point to Smart Server. + * + * smart.server.rpc.address + * 127.0.0.1:7042 + * + * + * 4. Restart HDFS + */ +public class SmartFileSystem extends DistributedFileSystem { + private SmartDFSClient smartDFSClient; + private boolean verifyChecksum = true; + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + + String rpcConfValue = conf.get(SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY); + if (rpcConfValue == null) { + throw new IOException("SmartServer address not found. Please configure " + + "it through " + SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY); + } + + String[] strings = rpcConfValue.split(":"); + InetSocketAddress smartServerAddress; + try { + smartServerAddress = new InetSocketAddress( + strings[strings.length - 2], + Integer.parseInt(strings[strings.length - 1])); + } catch (Exception e) { + throw new IOException("Incorrect SmartServer address. Please follow the " + + "IP/Hostname:Port format"); + } + this.smartDFSClient = new SmartDFSClient(conf, smartServerAddress); + } + + @Override + public FSDataInputStream open(Path path, final int bufferSize) + throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(path); + final DFSInputStream in = smartDFSClient.open( + absF.toUri().getPath(), bufferSize, verifyChecksum); + return smartDFSClient.createWrappedInputStream(in); + } + + @Override + public void setVerifyChecksum(boolean verifyChecksum) { + this.verifyChecksum = verifyChecksum; + } + + @Override + public FSDataOutputStream append(Path f, final int bufferSize, + final Progressable progress) throws IOException { + return append(f, EnumSet.of(CreateFlag.APPEND), bufferSize, progress); + } + + @Override + public FSDataOutputStream append(Path f, final EnumSet flag, + final int bufferSize, final Progressable progress) + throws IOException { + FSDataOutputStream out = super.append(f, flag, bufferSize, progress); + if (out.getPos() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(f)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Append", "SSM Small File")); + } + } + return out; + } + + @Override + public FSDataOutputStream append(Path f, final EnumSet flag, + final int bufferSize, final Progressable progress, + final InetSocketAddress[] favoredNodes) + throws IOException { + FSDataOutputStream out = super.append(f, flag, bufferSize, progress, favoredNodes); + if (out.getPos() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(f)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Append", "SSM Small File")); + } + } + return out; + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + FileStatus oldStatus = super.getFileStatus(f); + if (oldStatus != null && oldStatus.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(f)); + if (fileState instanceof CompactFileState) { + long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); + return new FileStatus(len, oldStatus.isDirectory(), oldStatus.getReplication(), + oldStatus.getBlockSize(), oldStatus.getModificationTime(), + oldStatus.getAccessTime(), oldStatus.getPermission(), + oldStatus.getOwner(), oldStatus.getGroup(), + oldStatus.isSymlink() ? oldStatus.getSymlink() : null, oldStatus.getPath()); + } + } + return oldStatus; + } + + @Override + public FileStatus[] listStatus(Path p) throws IOException { + FileStatus[] oldStatus = super.listStatus(p); + ArrayList newStatus = new ArrayList<>(oldStatus.length); + for (FileStatus status : oldStatus) { + if (status != null && status.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(status.getPath())); + if (fileState instanceof CompactFileState) { + long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); + newStatus.add(new FileStatus(len, status.isDirectory(), status.getReplication(), + status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), + status.getPermission(), status.getOwner(), status.getGroup(), + status.isSymlink() ? status.getSymlink() : null, status.getPath())); + } else { + newStatus.add(status); + } + } else { + newStatus.add(status); + } + } + return newStatus.toArray(new FileStatus[oldStatus.length]); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path p, final long start, + final long len) throws IOException { + BlockLocation[] blockLocations = super.getFileBlockLocations( + p, start, len); + if (blockLocations.length == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(p)); + if (fileState instanceof CompactFileState) { + FileContainerInfo fileContainerInfo = ((CompactFileState) fileState).getFileContainerInfo(); + String containerFile = fileContainerInfo.getContainerFilePath(); + long offset = fileContainerInfo.getOffset(); + blockLocations = super.getFileBlockLocations( + new Path(containerFile), offset + start, len); + for (BlockLocation blockLocation : blockLocations) { + blockLocation.setOffset(blockLocation.getOffset() - offset); + } + } + } + return blockLocations; + } + + @Override + public boolean setReplication(Path src, + final short replication) throws IOException { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Set replication", "SSM Small File")); + } else { + return super.setReplication(src, replication); + } + } + + @Override + public void setStoragePolicy(final Path src, final String policyName) + throws IOException { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Set storage policy", "SSM Small File")); + } else { + super.setStoragePolicy(src, policyName); + } + } + + @Override + public void concat(Path trg, Path [] psrcs) throws IOException { + try { + super.concat(trg, psrcs); + } catch (IOException e) { + for (Path src : psrcs) { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Concat", "SSM Small File")); + } + } + } + } + + + @Override + public FileStatus getFileLinkStatus(final Path f) throws IOException { + FileStatus fileStatus = super.getFileLinkStatus(f); + if (fileStatus.getLen() == 0) { + Path target = getLinkTarget(f); + FileState fileState = smartDFSClient.getFileState(getPathName(target)); + if (fileState instanceof CompactFileState) { + fileStatus = getFileStatus(target); + } + } + return fileStatus; + } + + @Override + public FileChecksum getFileChecksum(Path f) throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(f); + return new FileSystemLinkResolver() { + @Override + public FileChecksum doCall(final Path p) + throws IOException { + return smartDFSClient.getFileChecksum(getPathName(p), Long.MAX_VALUE); + } + + @Override + public FileChecksum next(final FileSystem fs, final Path p) + throws IOException { + return fs.getFileChecksum(p); + } + }.resolve(this, absF); + } + + @Override + public FileChecksum getFileChecksum(Path f, final long length) + throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(f); + return new FileSystemLinkResolver() { + @Override + public FileChecksum doCall(final Path p) + throws IOException { + return smartDFSClient.getFileChecksum(getPathName(p), length); + } + + @Override + public FileChecksum next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof SmartFileSystem) { + return fs.getFileChecksum(p, length); + } else { + throw new UnsupportedFileSystemException( + "getFileChecksum(Path, long) is not supported by " + + fs.getClass().getSimpleName()); + } + } + }.resolve(this, absF); + } + + @Override + public void setPermission(Path p, final FsPermission permission + ) throws IOException { + statistics.incrementWriteOps(1); + Path absF = fixRelativePart(p); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) + throws IOException, UnresolvedLinkException { + smartDFSClient.setPermission(getPathName(p), permission); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.setPermission(p, permission); + return null; + } + }.resolve(this, absF); + } + + @Override + public void setOwner(Path p, final String username, final String groupname + ) throws IOException { + if (username == null && groupname == null) { + throw new IOException("username == null && groupname == null"); + } + statistics.incrementWriteOps(1); + Path absF = fixRelativePart(p); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) + throws IOException, UnresolvedLinkException { + smartDFSClient.setOwner(getPathName(p), username, groupname); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.setOwner(p, username, groupname); + return null; + } + }.resolve(this, absF); + } + + @Override + public RemoteIterator listCorruptFileBlocks(Path path) + throws IOException { + RemoteIterator corruptFileBlocksIterator = super.listCorruptFileBlocks(path); + FileState fileState = smartDFSClient.getFileState(getPathName(path)); + if (fileState instanceof CompactFileState) { + corruptFileBlocksIterator = super.listCorruptFileBlocks( + new Path(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath())); + } + return corruptFileBlocksIterator; + } + + @Override + public void modifyAclEntries(Path path, final List aclSpec) + throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.modifyAclEntries(getPathName(p), aclSpec); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.modifyAclEntries(p, aclSpec); + return null; + } + }.resolve(this, absF); + } + + @Override + public void removeAclEntries(Path path, final List aclSpec) + throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.removeAclEntries(getPathName(p), aclSpec); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.removeAclEntries(p, aclSpec); + return null; + } + }.resolve(this, absF); + } + + @Override + public void removeDefaultAcl(Path path) throws IOException { + final Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.removeDefaultAcl(getPathName(p)); + return null; + } + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + fs.removeDefaultAcl(p); + return null; + } + }.resolve(this, absF); + } + + @Override + public void removeAcl(Path path) throws IOException { + final Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.removeAcl(getPathName(p)); + return null; + } + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + fs.removeAcl(p); + return null; + } + }.resolve(this, absF); + } + + @Override + public void setAcl(Path path, final List aclSpec) + throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.setAcl(getPathName(p), aclSpec); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.setAcl(p, aclSpec); + return null; + } + }.resolve(this, absF); + } + + @Override + public void createEncryptionZone(Path path, String keyName) + throws IOException { + smartDFSClient.createEncryptionZone(getPathName(path), keyName); + } + + @Override + public RemoteIterator listStatusIterator(final Path p) + throws IOException { + Path absF = fixRelativePart(p); + return new FileSystemLinkResolver>() { + @Override + public RemoteIterator doCall(final Path p) + throws IOException { + return new SmartDirListingIterator<>(p, false); + } + + @Override + public RemoteIterator next(final FileSystem fs, final Path p) + throws IOException { + return ((DistributedFileSystem) fs).listStatusIterator(p); + } + }.resolve(this, absF); + } + + @Override + protected RemoteIterator listLocatedStatus(final Path p, + final PathFilter filter) throws IOException { + Path absF = fixRelativePart(p); + return new FileSystemLinkResolver>() { + @Override + public RemoteIterator doCall(final Path p) + throws IOException { + return new SmartDirListingIterator<>(p, filter, true); + } + + @Override + public RemoteIterator next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof SmartFileSystem) { + return ((SmartFileSystem)fs).listLocatedStatus(p, filter); + } + // symlink resolution for this methods does not work cross file systems + // because it is a protected method. + throw new IOException("Link resolution does not work with multiple " + + "file systems for listLocatedStatus(): " + p); + } + }.resolve(this, absF); + } + + private class SmartDirListingIterator + implements RemoteIterator { + + private DirectoryListing thisListing; + private int i; + private Path p; + private String src; + private T curStat = null; + private PathFilter filter; + private boolean needLocation; + + private SmartDirListingIterator(Path p, PathFilter filter, + boolean needLocation) throws IOException { + this.p = p; + this.src = getPathName(p); + this.filter = filter; + this.needLocation = needLocation; + // fetch the first batch of entries in the directory + thisListing = smartDFSClient.listPaths(src, HdfsFileStatus.EMPTY_NAME, + needLocation); + statistics.incrementReadOps(1); + // the directory does not exist + if (thisListing == null) { + throw new FileNotFoundException("File " + p + " does not exist."); + } + i = 0; + } + + private SmartDirListingIterator(Path p, boolean needLocation) + throws IOException { + this(p, null, needLocation); + } + + @Override + @SuppressWarnings("unchecked") + public boolean hasNext() throws IOException { + while (curStat == null && hasNextNoFilter()) { + T next; + HdfsFileStatus fileStat = thisListing.getPartialListing()[i++]; + if (needLocation) { + next = (T)((HdfsLocatedFileStatus) fileStat).makeQualifiedLocated(getUri(), p); + String fileName = next.getPath().toUri().getPath(); + + // Reconstruct FileStatus + if (next.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(fileName); + if (fileState instanceof CompactFileState) { + CompactFileState compactFileState = (CompactFileState) fileState; + long len = compactFileState.getFileContainerInfo().getLength(); + BlockLocation[] blockLocations = smartDFSClient.getBlockLocations( + fileName, 0, len); + next = (T) new LocatedFileStatus(len, + next.isDirectory(), + next.getReplication(), + next.getBlockSize(), + next.getModificationTime(), + next.getAccessTime(), + next.getPermission(), + next.getOwner(), + next.getGroup(), + next.isSymlink() ? next.getSymlink() : null, + next.getPath(), + blockLocations); + } + } + } else { + next = (T) fileStat.makeQualified(getUri(), p); + String fileName = next.getPath().toUri().getPath(); + + // Reconstruct FileStatus + if (next.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(fileName); + if (fileState instanceof CompactFileState) { + CompactFileState compactFileState = (CompactFileState) fileState; + long len = compactFileState.getFileContainerInfo().getLength(); + next = (T) new FileStatus(len, + next.isDirectory(), + next.getReplication(), + next.getBlockSize(), + next.getModificationTime(), + next.getAccessTime(), + next.getPermission(), + next.getOwner(), + next.getGroup(), + next.isSymlink() ? next.getSymlink() : null, + next.getPath()); + } + } + } + + // apply filter if not null + if (filter == null || filter.accept(next.getPath())) { + curStat = next; + } + } + return curStat != null; + } + + /** + * Check if there is a next item before applying the given filter + */ + private boolean hasNextNoFilter() throws IOException { + if (thisListing == null) { + return false; + } + if (i >= thisListing.getPartialListing().length && thisListing.hasMore()) { + // current listing is exhausted & fetch a new listing + thisListing = smartDFSClient.listPaths( + src, thisListing.getLastName(), needLocation); + statistics.incrementReadOps(1); + if (thisListing == null) { + return false; + } + i = 0; + } + return (i < thisListing.getPartialListing().length); + } + + @Override + public T next() throws IOException { + if (hasNext()) { + T tmp = curStat; + curStat = null; + return tmp; + } + throw new java.util.NoSuchElementException("No more entry in " + p); + } + } + + @Override + public boolean isFileClosed(final Path src) throws IOException { + boolean isFileClosed = super.isFileClosed(src); + if (!isFileClosed) { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + isFileClosed = smartDFSClient.isFileClosed(containerFile); + } + } + return isFileClosed; + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + if (smartDFSClient != null) { + this.smartDFSClient.close(); + } + } + } + + /** + * Checks that the passed URI belongs to this filesystem and returns + * just the path component. Expects a URI with an absolute path. + * + * @param file URI with absolute path + * @return path component of {file} + * @throws IllegalArgumentException if URI does not belong to this DFS + */ + private String getPathName(Path file) { + checkPath(file); + String result = fixRelativePart(file).toUri().getPath(); + if (!DFSUtil.isValidName(result)) { + throw new IllegalArgumentException("Pathname " + result + " from " + + file + " is not a valid DFS filename."); + } + return result; + } +} diff --git a/smart-hadoop-support/smart-hadoop-client-3.1/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java b/smart-hadoop-support/smart-hadoop-client-3.1/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java new file mode 100644 index 00000000000..6b1de8cb9a1 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-client-3.1/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java @@ -0,0 +1,548 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.client; + +import org.apache.commons.lang.SerializationUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.SmartInputStreamFactory; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsPathHandle; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Progressable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.smartdata.SmartConstants; +import org.smartdata.client.SmartClient; +import org.smartdata.hdfs.CompatibilityHelperLoader; +import org.smartdata.metrics.FileAccessEvent; +import org.smartdata.model.CompactFileState; +import org.smartdata.model.FileState; +import org.smartdata.model.NormalFileState; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.EnumSet; +import java.util.List; + +public class SmartDFSClient extends DFSClient { + private static final Logger LOG = LoggerFactory.getLogger(SmartDFSClient.class); + private static final String CALLER_CLASS = "org.apache.hadoop.hdfs.DFSClient"; + private SmartClient smartClient = null; + private boolean healthy = false; + + public SmartDFSClient(InetSocketAddress nameNodeAddress, Configuration conf, + InetSocketAddress smartServerAddress) throws IOException { + super(nameNodeAddress, conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(final URI nameNodeUri, final Configuration conf, + final InetSocketAddress smartServerAddress) throws IOException { + super(nameNodeUri, conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(URI nameNodeUri, Configuration conf, + FileSystem.Statistics stats, InetSocketAddress smartServerAddress) + throws IOException { + super(nameNodeUri, conf, stats); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(Configuration conf, + InetSocketAddress smartServerAddress) throws IOException { + super(conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(Configuration conf) throws IOException { + super(conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + @Override + public DFSInputStream open(String src) throws IOException { + return open(src, 4096, true); + } + + @Override + public DFSInputStream open(String src, int buffersize, + boolean verifyChecksum) throws IOException { + DFSInputStream is = super.open(src, buffersize, verifyChecksum); + if (is.getFileLength() == 0) { + is.close(); + FileState fileState = getFileState(src); + if (fileState.getFileStage().equals(FileState.FileStage.PROCESSING)) { + throw new IOException("Cannot open " + src + " when it is under PROCESSING to " + + fileState.getFileType()); + } + is = SmartInputStreamFactory.create(this, src, + verifyChecksum, fileState); + } + reportFileAccessEvent(src); + return is; + } + + @Deprecated + @Override + public DFSInputStream open(String src, int buffersize, + boolean verifyChecksum, FileSystem.Statistics stats) + throws IOException { + return open(src, buffersize, verifyChecksum); + } + + @Override + public DFSInputStream open(HdfsPathHandle fd, int buffersize, boolean verifyChecksum) throws IOException { + String src = fd.getPath(); + DFSInputStream is = super.open(fd, buffersize, verifyChecksum); + if (is.getFileLength() == 0) { + is.close(); + FileState fileState = getFileState(src); + if (fileState.getFileStage().equals(FileState.FileStage.PROCESSING)) { + throw new IOException("Cannot open " + src + " when it is under PROCESSING to " + + fileState.getFileType()); + } + is = SmartInputStreamFactory.create(this, src, + verifyChecksum, fileState); + } + reportFileAccessEvent(src); + return is; + } + + @Override + public HdfsDataOutputStream append(final String src, final int buffersize, + EnumSet flag, final Progressable progress, + final FileSystem.Statistics statistics) throws IOException { + HdfsDataOutputStream out = super.append(src, buffersize, flag, progress, statistics); + if (out.getPos() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + out.close(); + throw new IOException(getExceptionMsg("Append", "SSM Small File")); + } + } + return out; + } + + @Override + public HdfsDataOutputStream append(final String src, final int buffersize, + EnumSet flag, final Progressable progress, + final FileSystem.Statistics statistics, + final InetSocketAddress[] favoredNodes) throws IOException { + HdfsDataOutputStream out = super.append( + src, buffersize, flag, progress, statistics, favoredNodes); + if (out.getPos() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + out.close(); + throw new IOException(getExceptionMsg("Append", "SSM Small File")); + } + } + return out; + } + + @Override + public HdfsFileStatus getFileInfo(String src) throws IOException { + HdfsFileStatus oldStatus = super.getFileInfo(src); + if (oldStatus != null && oldStatus.getLen() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); + return CompatibilityHelperLoader.getHelper().createHdfsFileStatus(len, oldStatus.isDir(), oldStatus.getReplication(), + oldStatus.getBlockSize(), oldStatus.getModificationTime(), oldStatus.getAccessTime(), + oldStatus.getPermission(), oldStatus.getOwner(), oldStatus.getGroup(), + oldStatus.isSymlink() ? oldStatus.getSymlinkInBytes() : null, + oldStatus.isEmptyLocalName() ? new byte[0] : oldStatus.getLocalNameInBytes(), + oldStatus.getFileId(), oldStatus.getChildrenNum(), + oldStatus.getFileEncryptionInfo(), oldStatus.getStoragePolicy()); + } + } + return oldStatus; + } + + @Override + public LocatedBlocks getLocatedBlocks(String src, long start) + throws IOException { + LocatedBlocks locatedBlocks = super.getLocatedBlocks(src, start); + if (!CALLER_CLASS.equals(Thread.currentThread().getStackTrace()[2].getClassName()) + && locatedBlocks.getFileLength() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset(); + return super.getLocatedBlocks(containerFile, offset + start); + } + } + return locatedBlocks; + } + + @Override + public BlockLocation[] getBlockLocations(String src, long start, + long length) throws IOException { + BlockLocation[] blockLocations = super.getBlockLocations(src, start, length); + if (blockLocations.length == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset(); + blockLocations = super.getBlockLocations(containerFile, offset + start, length); + for (BlockLocation blockLocation : blockLocations) { + blockLocation.setOffset(blockLocation.getOffset() - offset); + } + return blockLocations; + } + } + return blockLocations; + } + + @Override + public boolean setReplication(String src, short replication) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set replication", "SSM Small File")); + } else { + return super.setReplication(src, replication); + } + } + + @Override + public void setStoragePolicy(String src, String policyName) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set storage policy", "SSM Small File")); + } else { + super.setStoragePolicy(src, policyName); + } + } + + @Override + public long getBlockSize(String f) throws IOException { + long blockSize = super.getBlockSize(f); + FileState fileState = getFileState(f); + if (fileState instanceof CompactFileState) { + blockSize = super.getBlockSize(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath()); + } + return blockSize; + } + + @Override + public void concat(String trg, String [] srcs) throws IOException { + try { + super.concat(trg, srcs); + } catch (IOException e) { + for (String src : srcs) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Concat", "SSM Small File")); + } + } + throw e; + } + } + + @Override + public HdfsFileStatus getFileLinkInfo(String src) throws IOException { + HdfsFileStatus fileStatus = super.getFileLinkInfo(src); + if (fileStatus.getLen() == 0) { + String target = super.getLinkTarget(src); + FileState fileState = getFileState(target); + if (fileState instanceof CompactFileState) { + fileStatus = getFileInfo(target); + } + } + return fileStatus; + } + + @Override + public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) + throws IOException { + MD5MD5CRC32FileChecksum ret = super.getFileChecksum(src, length); + if (ret.getChecksumOpt().getBytesPerChecksum() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Get file checksum", "SSM Small File")); + } + } + return ret; + } + + @Override + public void setPermission(String src, FsPermission permission) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set permission", "SSM Small File")); + } else { + super.setPermission(src, permission); + } + } + + @Override + public void setOwner(String src, String username, String groupname) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set owner", "SSM Small File")); + } else { + super.setOwner(src, username, groupname); + } + } + + @Override + public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie) + throws IOException { + CorruptFileBlocks corruptFileBlocks = super.listCorruptFileBlocks(path, cookie); + FileState fileState = getFileState(path); + if (fileState instanceof CompactFileState) { + corruptFileBlocks = super.listCorruptFileBlocks(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(), cookie); + } + return corruptFileBlocks; + } + + @Override + public void modifyAclEntries(String src, List aclSpec) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Modify acl entries", "SSM Small File")); + } else { + super.modifyAclEntries(src, aclSpec); + } + } + + @Override + public void removeAclEntries(String src, List aclSpec) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Remove acl entries", "SSM Small File")); + } else { + super.removeAclEntries(src, aclSpec); + } + } + + @Override + public void removeDefaultAcl(String src) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Remove default acl", "SSM Small File")); + } else { + super.removeDefaultAcl(src); + } + } + + @Override + public void removeAcl(String src) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Remove acl", "SSM Small File")); + } else { + super.removeAcl(src); + } + } + + @Override + public void setAcl(String src, List aclSpec) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set acl", "SSM Small File")); + } else { + super.setAcl(src, aclSpec); + } + } + + @Override + public void createEncryptionZone(String src, String keyName) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Create encryption zone", "SSM Small File")); + } else { + super.createEncryptionZone(src, keyName); + } + } + + @Override + public void checkAccess(String src, FsAction mode) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + super.checkAccess(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(), mode); + } else { + super.checkAccess(src, mode); + } + } + + @Override + public boolean isFileClosed(String src) throws IOException { + boolean isFileClosed = super.isFileClosed(src); + if (!isFileClosed) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + isFileClosed = super.isFileClosed(containerFile); + } + } + return isFileClosed; + } + + @Override + public synchronized void close() throws IOException { + try { + super.close(); + } finally { + try { + if (smartClient != null) { + smartClient.close(); + } + } finally { + healthy = false; + } + } + } + + /** + * Report file access event to SSM server. + */ + private void reportFileAccessEvent(String src) { + try { + if (!healthy) { + return; + } + String userName; + try { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } catch (IOException e) { + userName = "Unknown"; + } + smartClient.reportFileAccessEvent(new FileAccessEvent(src, userName)); + } catch (IOException e) { + // Here just ignores that failed to report + LOG.error("Cannot report file access event to SmartServer: " + src + + " , for: " + e.getMessage() + + " , report mechanism will be disabled now in this instance."); + healthy = false; + } + } + + /** + * Check if the smart client is disabled. + */ + private boolean isSmartClientDisabled() { + File idFile = new File(SmartConstants.SMART_CLIENT_DISABLED_ID_FILE); + return idFile.exists(); + } + + /** + * Get the exception message of unsupported operation. + * + * @param operation the hdfs operation name + * @param fileType the type of SSM specify file + * @return the message of unsupported exception + */ + public String getExceptionMsg(String operation, String fileType) { + return String.format("%s is not supported for %s.", operation, fileType); + } + + /** + * Get file state of the specified file. + * + * @param filePath the path of source file + * @return file state of source file + * @throws IOException e + */ + public FileState getFileState(String filePath) throws IOException { + try { + byte[] fileState = getXAttr(filePath, SmartConstants.SMART_FILE_STATE_XATTR_NAME); + if (fileState != null) { + return (FileState) SerializationUtils.deserialize(fileState); + } + } catch (RemoteException e) { + return new NormalFileState(filePath); + } + + return new NormalFileState(filePath); + } +} diff --git a/smart-hadoop-support/smart-hadoop-client-cdh-2.6/pom.xml b/smart-hadoop-support/smart-hadoop-client-cdh-2.6/pom.xml new file mode 100644 index 00000000000..9e683217fb6 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-client-cdh-2.6/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + + org.smartdata + smart-hadoop-support + 1.5.0-SNAPSHOT + .. + + + smart-hadoop-client-cdh-2.6 + 1.5.0-SNAPSHOT + jar + + + + org.smartdata + smart-common + 1.5.0-SNAPSHOT + provided + + + org.smartdata + smart-metrics + 1.5.0-SNAPSHOT + provided + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + provided + + + org.smartdata + smart-client + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop + 1.5.0-SNAPSHOT + provided + + + org.smartdata + smart-inputstream + 1.5.0-SNAPSHOT + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + package + + copy-dependencies + + + target/lib + + + + + + + diff --git a/smart-hadoop-support/smart-hadoop-client-cdh-2.6/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java b/smart-hadoop-support/smart-hadoop-client-cdh-2.6/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java new file mode 100644 index 00000000000..2175de1a697 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-client-cdh-2.6/src/main/java/org/smartdata/hadoop/filesystem/SmartFileSystem.java @@ -0,0 +1,705 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hadoop.filesystem; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemLinkResolver; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; +import org.apache.hadoop.util.Progressable; +import org.smartdata.conf.SmartConfKeys; +import org.smartdata.hdfs.client.SmartDFSClient; +import org.smartdata.model.CompactFileState; +import org.smartdata.model.FileContainerInfo; +import org.smartdata.model.FileState; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +/** + * SmartFileSystem Deploy Guide + * 1. Build SSM, get all jar files start with name Smart* + * 2. Copy these jar files to HDFS classpath + * 3. Reconfigure HDFS + * Please do the following configurations, + * 1. core-site.xml + * Change property "fs.hdfs.impl" value, to point to the Smart Server provided + * "Smart File System". + * + * fs.hdfs.impl + * org.smartdata.hadoop.filesystem.SmartFileSystem + * The FileSystem for hdfs URL + * + * 2. hdfs-site.xml + * Add property "smart.server.rpc.adddress" to point to Smart Server. + * + * smart.server.rpc.address + * 127.0.0.1:7042 + * + * + * 4. Restart HDFS + */ +public class SmartFileSystem extends DistributedFileSystem { + private SmartDFSClient smartDFSClient; + private boolean verifyChecksum = true; + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + + String rpcConfValue = conf.get(SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY); + if (rpcConfValue == null) { + throw new IOException("SmartServer address not found. Please configure " + + "it through " + SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY); + } + + String[] strings = rpcConfValue.split(":"); + InetSocketAddress smartServerAddress; + try { + smartServerAddress = new InetSocketAddress( + strings[strings.length - 2], + Integer.parseInt(strings[strings.length - 1])); + } catch (Exception e) { + throw new IOException("Incorrect SmartServer address. Please follow the " + + "IP/Hostname:Port format"); + } + this.smartDFSClient = new SmartDFSClient(conf, smartServerAddress); + } + + @Override + public FSDataInputStream open(Path path, final int bufferSize) + throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(path); + final DFSInputStream in = smartDFSClient.open( + absF.toUri().getPath(), bufferSize, verifyChecksum); + return smartDFSClient.createWrappedInputStream(in); + } + + @Override + public void setVerifyChecksum(boolean verifyChecksum) { + this.verifyChecksum = verifyChecksum; + } + +// @Override +// public FSDataOutputStream append(Path f, final int bufferSize, +// final Progressable progress) throws IOException { +// try { +// return super.append(f, bufferSize, progress); +// } catch (IOException e) { +// FileState fileState = smartDFSClient.getFileState(getPathName(f)); +// if (fileState instanceof CompactFileState) { +// throw new IOException( +// smartDFSClient.getExceptionMsg("Append", "SSM Small File")); +// } +// throw e; +// } +// } + + @Override + public FSDataOutputStream append(Path f, final int bufferSize, final Progressable progress) + throws IOException { + FSDataOutputStream out = super.append(f, bufferSize, progress); + if (out.getPos() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(f)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Append", "SSM Small File")); + } + } + return out; + } + +// @Override +// public FSDataOutputStream append(Path f, final EnumSet flag, +// final int bufferSize, final Progressable progress, +// final InetSocketAddress[] favoredNodes) +// throws IOException { +// FSDataOutputStream out = super.append(f, flag, bufferSize, progress, favoredNodes); +// if (out.getPos() == 0) { +// FileState fileState = smartDFSClient.getFileState(getPathName(f)); +// if (fileState instanceof CompactFileState) { +// throw new IOException( +// smartDFSClient.getExceptionMsg("Append", "SSM Small File")); +// } +// } +// return out; +// } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + FileStatus oldStatus = super.getFileStatus(f); + if (oldStatus != null && oldStatus.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(f)); + if (fileState instanceof CompactFileState) { + long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); + return new FileStatus(len, oldStatus.isDirectory(), oldStatus.getReplication(), + oldStatus.getBlockSize(), oldStatus.getModificationTime(), + oldStatus.getAccessTime(), oldStatus.getPermission(), + oldStatus.getOwner(), oldStatus.getGroup(), + oldStatus.isSymlink() ? oldStatus.getSymlink() : null, oldStatus.getPath()); + } + } + return oldStatus; + } + + @Override + public FileStatus[] listStatus(Path p) throws IOException { + FileStatus[] oldStatus = super.listStatus(p); + ArrayList newStatus = new ArrayList<>(oldStatus.length); + for (FileStatus status : oldStatus) { + if (status != null && status.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(status.getPath())); + if (fileState instanceof CompactFileState) { + long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); + newStatus.add(new FileStatus(len, status.isDirectory(), status.getReplication(), + status.getBlockSize(), status.getModificationTime(), status.getAccessTime(), + status.getPermission(), status.getOwner(), status.getGroup(), + status.isSymlink() ? status.getSymlink() : null, status.getPath())); + } else { + newStatus.add(status); + } + } else { + newStatus.add(status); + } + } + return newStatus.toArray(new FileStatus[oldStatus.length]); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path p, final long start, + final long len) throws IOException { + BlockLocation[] blockLocations = super.getFileBlockLocations( + p, start, len); + if (blockLocations.length == 0) { + FileState fileState = smartDFSClient.getFileState(getPathName(p)); + if (fileState instanceof CompactFileState) { + FileContainerInfo fileContainerInfo = ((CompactFileState) fileState).getFileContainerInfo(); + String containerFile = fileContainerInfo.getContainerFilePath(); + long offset = fileContainerInfo.getOffset(); + blockLocations = super.getFileBlockLocations( + new Path(containerFile), offset + start, len); + for (BlockLocation blockLocation : blockLocations) { + blockLocation.setOffset(blockLocation.getOffset() - offset); + } + } + } + return blockLocations; + } + + @Override + public boolean setReplication(Path src, + final short replication) throws IOException { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Set replication", "SSM Small File")); + } else { + return super.setReplication(src, replication); + } + } + + @Override + public void setStoragePolicy(final Path src, final String policyName) + throws IOException { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Set storage policy", "SSM Small File")); + } else { + super.setStoragePolicy(src, policyName); + } + } + + @Override + public void concat(Path trg, Path [] psrcs) throws IOException { + try { + super.concat(trg, psrcs); + } catch (IOException e) { + for (Path src : psrcs) { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + throw new IOException( + smartDFSClient.getExceptionMsg("Concat", "SSM Small File")); + } + } + } + } + + + @Override + public FileStatus getFileLinkStatus(final Path f) throws IOException { + FileStatus fileStatus = super.getFileLinkStatus(f); + if (fileStatus.getLen() == 0) { + Path target = getLinkTarget(f); + FileState fileState = smartDFSClient.getFileState(getPathName(target)); + if (fileState instanceof CompactFileState) { + fileStatus = getFileStatus(target); + } + } + return fileStatus; + } + + @Override + public FileChecksum getFileChecksum(Path f) throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(f); + return new FileSystemLinkResolver() { + @Override + public FileChecksum doCall(final Path p) + throws IOException { + return smartDFSClient.getFileChecksum(getPathName(p), Long.MAX_VALUE); + } + + @Override + public FileChecksum next(final FileSystem fs, final Path p) + throws IOException { + return fs.getFileChecksum(p); + } + }.resolve(this, absF); + } + + @Override + public FileChecksum getFileChecksum(Path f, final long length) + throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(f); + return new FileSystemLinkResolver() { + @Override + public FileChecksum doCall(final Path p) + throws IOException { + return smartDFSClient.getFileChecksum(getPathName(p), length); + } + + @Override + public FileChecksum next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof SmartFileSystem) { + return fs.getFileChecksum(p, length); + } else { + throw new UnsupportedFileSystemException( + "getFileChecksum(Path, long) is not supported by " + + fs.getClass().getSimpleName()); + } + } + }.resolve(this, absF); + } + + @Override + public void setPermission(Path p, final FsPermission permission + ) throws IOException { + statistics.incrementWriteOps(1); + Path absF = fixRelativePart(p); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) + throws IOException, UnresolvedLinkException { + smartDFSClient.setPermission(getPathName(p), permission); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.setPermission(p, permission); + return null; + } + }.resolve(this, absF); + } + + @Override + public void setOwner(Path p, final String username, final String groupname + ) throws IOException { + if (username == null && groupname == null) { + throw new IOException("username == null && groupname == null"); + } + statistics.incrementWriteOps(1); + Path absF = fixRelativePart(p); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) + throws IOException, UnresolvedLinkException { + smartDFSClient.setOwner(getPathName(p), username, groupname); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.setOwner(p, username, groupname); + return null; + } + }.resolve(this, absF); + } + + @Override + public RemoteIterator listCorruptFileBlocks(Path path) + throws IOException { + RemoteIterator corruptFileBlocksIterator = super.listCorruptFileBlocks(path); + FileState fileState = smartDFSClient.getFileState(getPathName(path)); + if (fileState instanceof CompactFileState) { + corruptFileBlocksIterator = super.listCorruptFileBlocks( + new Path(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath())); + } + return corruptFileBlocksIterator; + } + + @Override + public void modifyAclEntries(Path path, final List aclSpec) + throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.modifyAclEntries(getPathName(p), aclSpec); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.modifyAclEntries(p, aclSpec); + return null; + } + }.resolve(this, absF); + } + + @Override + public void removeAclEntries(Path path, final List aclSpec) + throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.removeAclEntries(getPathName(p), aclSpec); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.removeAclEntries(p, aclSpec); + return null; + } + }.resolve(this, absF); + } + + @Override + public void removeDefaultAcl(Path path) throws IOException { + final Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.removeDefaultAcl(getPathName(p)); + return null; + } + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + fs.removeDefaultAcl(p); + return null; + } + }.resolve(this, absF); + } + + @Override + public void removeAcl(Path path) throws IOException { + final Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.removeAcl(getPathName(p)); + return null; + } + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + fs.removeAcl(p); + return null; + } + }.resolve(this, absF); + } + + @Override + public void setAcl(Path path, final List aclSpec) + throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(final Path p) throws IOException { + smartDFSClient.setAcl(getPathName(p), aclSpec); + return null; + } + + @Override + public Void next(final FileSystem fs, final Path p) + throws IOException { + fs.setAcl(p, aclSpec); + return null; + } + }.resolve(this, absF); + } + + @Override + public void createEncryptionZone(Path path, String keyName) + throws IOException { + smartDFSClient.createEncryptionZone(getPathName(path), keyName); + } + + @Override + public RemoteIterator listStatusIterator(final Path p) + throws IOException { + Path absF = fixRelativePart(p); + return new FileSystemLinkResolver>() { + @Override + public RemoteIterator doCall(final Path p) + throws IOException { + return new SmartDirListingIterator<>(p, false); + } + + @Override + public RemoteIterator next(final FileSystem fs, final Path p) + throws IOException { + return ((DistributedFileSystem) fs).listStatusIterator(p); + } + }.resolve(this, absF); + } + + @Override + protected RemoteIterator listLocatedStatus(final Path p, + final PathFilter filter) throws IOException { + Path absF = fixRelativePart(p); + return new FileSystemLinkResolver>() { + @Override + public RemoteIterator doCall(final Path p) + throws IOException { + return new SmartDirListingIterator<>(p, filter, true); + } + + @Override + public RemoteIterator next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof SmartFileSystem) { + return ((SmartFileSystem)fs).listLocatedStatus(p, filter); + } + // symlink resolution for this methods does not work cross file systems + // because it is a protected method. + throw new IOException("Link resolution does not work with multiple " + + "file systems for listLocatedStatus(): " + p); + } + }.resolve(this, absF); + } + + private class SmartDirListingIterator + implements RemoteIterator { + + private DirectoryListing thisListing; + private int i; + private Path p; + private String src; + private T curStat = null; + private PathFilter filter; + private boolean needLocation; + + private SmartDirListingIterator(Path p, PathFilter filter, + boolean needLocation) throws IOException { + this.p = p; + this.src = getPathName(p); + this.filter = filter; + this.needLocation = needLocation; + // fetch the first batch of entries in the directory + thisListing = smartDFSClient.listPaths(src, HdfsFileStatus.EMPTY_NAME, + needLocation); + statistics.incrementReadOps(1); + // the directory does not exist + if (thisListing == null) { + throw new FileNotFoundException("File " + p + " does not exist."); + } + i = 0; + } + + private SmartDirListingIterator(Path p, boolean needLocation) + throws IOException { + this(p, null, needLocation); + } + + @Override + @SuppressWarnings("unchecked") + public boolean hasNext() throws IOException { + while (curStat == null && hasNextNoFilter()) { + T next; + HdfsFileStatus fileStat = thisListing.getPartialListing()[i++]; + if (needLocation) { + next = (T)((HdfsLocatedFileStatus) fileStat).makeQualifiedLocated(getUri(), p); + String fileName = next.getPath().toUri().getPath(); + + // Reconstruct FileStatus + if (next.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(fileName); + if (fileState instanceof CompactFileState) { + CompactFileState compactFileState = (CompactFileState) fileState; + long len = compactFileState.getFileContainerInfo().getLength(); + BlockLocation[] blockLocations = smartDFSClient.getBlockLocations( + fileName, 0, len); + next = (T) new LocatedFileStatus(len, + next.isDirectory(), + next.getReplication(), + next.getBlockSize(), + next.getModificationTime(), + next.getAccessTime(), + next.getPermission(), + next.getOwner(), + next.getGroup(), + next.isSymlink() ? next.getSymlink() : null, + next.getPath(), + blockLocations); + } + } + } else { + next = (T) fileStat.makeQualified(getUri(), p); + String fileName = next.getPath().toUri().getPath(); + + // Reconstruct FileStatus + if (next.getLen() == 0) { + FileState fileState = smartDFSClient.getFileState(fileName); + if (fileState instanceof CompactFileState) { + CompactFileState compactFileState = (CompactFileState) fileState; + long len = compactFileState.getFileContainerInfo().getLength(); + next = (T) new FileStatus(len, + next.isDirectory(), + next.getReplication(), + next.getBlockSize(), + next.getModificationTime(), + next.getAccessTime(), + next.getPermission(), + next.getOwner(), + next.getGroup(), + next.isSymlink() ? next.getSymlink() : null, + next.getPath()); + } + } + } + + // apply filter if not null + if (filter == null || filter.accept(next.getPath())) { + curStat = next; + } + } + return curStat != null; + } + + /** + * Check if there is a next item before applying the given filter + */ + private boolean hasNextNoFilter() throws IOException { + if (thisListing == null) { + return false; + } + if (i >= thisListing.getPartialListing().length && thisListing.hasMore()) { + // current listing is exhausted & fetch a new listing + thisListing = smartDFSClient.listPaths( + src, thisListing.getLastName(), needLocation); + statistics.incrementReadOps(1); + if (thisListing == null) { + return false; + } + i = 0; + } + return (i < thisListing.getPartialListing().length); + } + + @Override + public T next() throws IOException { + if (hasNext()) { + T tmp = curStat; + curStat = null; + return tmp; + } + throw new java.util.NoSuchElementException("No more entry in " + p); + } + } + + @Override + public boolean isFileClosed(final Path src) throws IOException { + boolean isFileClosed = super.isFileClosed(src); + if (!isFileClosed) { + FileState fileState = smartDFSClient.getFileState(getPathName(src)); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + isFileClosed = smartDFSClient.isFileClosed(containerFile); + } + } + return isFileClosed; + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + if (smartDFSClient != null) { + this.smartDFSClient.close(); + } + } + } + + /** + * Checks that the passed URI belongs to this filesystem and returns + * just the path component. Expects a URI with an absolute path. + * + * @param file URI with absolute path + * @return path component of {file} + * @throws IllegalArgumentException if URI does not belong to this DFS + */ + private String getPathName(Path file) { + checkPath(file); + String result = fixRelativePart(file).toUri().getPath(); + if (!DFSUtil.isValidName(result)) { + throw new IllegalArgumentException("Pathname " + result + " from " + + file + " is not a valid DFS filename."); + } + return result; + } +} diff --git a/smart-hadoop-support/smart-hadoop-client-cdh-2.6/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java b/smart-hadoop-support/smart-hadoop-client-cdh-2.6/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java new file mode 100644 index 00000000000..25f9778dccd --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-client-cdh-2.6/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java @@ -0,0 +1,511 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.client; + +import org.apache.commons.lang.SerializationUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.SmartInputStreamFactory; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Progressable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.smartdata.SmartConstants; +import org.smartdata.client.SmartClient; +import org.smartdata.hdfs.CompatibilityHelperLoader; +import org.smartdata.metrics.FileAccessEvent; +import org.smartdata.model.CompactFileState; +import org.smartdata.model.FileState; +import org.smartdata.model.NormalFileState; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.EnumSet; +import java.util.List; + +public class SmartDFSClient extends DFSClient { + private static final Logger LOG = LoggerFactory.getLogger(SmartDFSClient.class); + private static final String CALLER_CLASS = "org.apache.hadoop.hdfs.DFSInputStream"; + private SmartClient smartClient = null; + private boolean healthy = false; + + public SmartDFSClient(InetSocketAddress nameNodeAddress, Configuration conf, + InetSocketAddress smartServerAddress) throws IOException { + super(nameNodeAddress, conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(final URI nameNodeUri, final Configuration conf, + final InetSocketAddress smartServerAddress) throws IOException { + super(nameNodeUri, conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(URI nameNodeUri, Configuration conf, + FileSystem.Statistics stats, InetSocketAddress smartServerAddress) + throws IOException { + super(nameNodeUri, conf, stats); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(Configuration conf, + InetSocketAddress smartServerAddress) throws IOException { + super(conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf, smartServerAddress); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + public SmartDFSClient(Configuration conf) throws IOException { + super(conf); + if (isSmartClientDisabled()) { + return; + } + try { + smartClient = new SmartClient(conf); + healthy = true; + } catch (IOException e) { + super.close(); + throw e; + } + } + + @Override + public DFSInputStream open(String src) throws IOException { + return open(src, 4096, true); + } + + @Override + public DFSInputStream open(String src, int buffersize, + boolean verifyChecksum) throws IOException { + DFSInputStream is = super.open(src, buffersize, verifyChecksum); + if (is.getFileLength() == 0) { + is.close(); + FileState fileState = getFileState(src); + if (fileState.getFileStage().equals(FileState.FileStage.PROCESSING)) { + throw new IOException("Cannot open " + src + " when it is under PROCESSING to " + + fileState.getFileType()); + } + is = SmartInputStreamFactory.create(this, src, + verifyChecksum, fileState); + } + reportFileAccessEvent(src); + return is; + } + + @Deprecated + @Override + public DFSInputStream open(String src, int buffersize, + boolean verifyChecksum, FileSystem.Statistics stats) + throws IOException { + return open(src, buffersize, verifyChecksum); + } + + @Override + public HdfsDataOutputStream append(final String src, final int buffersize, + final Progressable progress, final FileSystem.Statistics statistics) throws IOException { + HdfsDataOutputStream out = super.append(src, buffersize, progress, statistics); + if (out.getPos() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + out.close(); + throw new IOException(getExceptionMsg("Append", "SSM Small File")); + } + } + return out; + } + + @Override + public HdfsFileStatus getFileInfo(String src) throws IOException { + HdfsFileStatus oldStatus = super.getFileInfo(src); + if (oldStatus != null && oldStatus.getLen() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + long len = ((CompactFileState) fileState).getFileContainerInfo().getLength(); + return CompatibilityHelperLoader.getHelper().createHdfsFileStatus(len, oldStatus.isDir(), oldStatus.getReplication(), + oldStatus.getBlockSize(), oldStatus.getModificationTime(), oldStatus.getAccessTime(), + oldStatus.getPermission(), oldStatus.getOwner(), oldStatus.getGroup(), + oldStatus.isSymlink() ? oldStatus.getSymlinkInBytes() : null, + oldStatus.isEmptyLocalName() ? new byte[0] : oldStatus.getLocalNameInBytes(), + oldStatus.getFileId(), oldStatus.getChildrenNum(), + oldStatus.getFileEncryptionInfo(), oldStatus.getStoragePolicy()); + } + } + return oldStatus; + } + + @Override + public LocatedBlocks getLocatedBlocks(String src, long start) + throws IOException { + LocatedBlocks locatedBlocks = super.getLocatedBlocks(src, start); + if (!CALLER_CLASS.equals(Thread.currentThread().getStackTrace()[2].getClassName()) + && locatedBlocks.getFileLength() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset(); + return super.getLocatedBlocks(containerFile, offset + start); + } + } + return locatedBlocks; + } + + @Override + public BlockLocation[] getBlockLocations(String src, long start, + long length) throws IOException { + BlockLocation[] blockLocations = super.getBlockLocations(src, start, length); + if (blockLocations.length == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + long offset = ((CompactFileState) fileState).getFileContainerInfo().getOffset(); + blockLocations = super.getBlockLocations(containerFile, offset + start, length); + for (BlockLocation blockLocation : blockLocations) { + blockLocation.setOffset(blockLocation.getOffset() - offset); + } + return blockLocations; + } + } + return blockLocations; + } + + @Override + public boolean setReplication(String src, short replication) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set replication", "SSM Small File")); + } else { + return super.setReplication(src, replication); + } + } + + @Override + public void setStoragePolicy(String src, String policyName) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set storage policy", "SSM Small File")); + } else { + super.setStoragePolicy(src, policyName); + } + } + + @Override + public long getBlockSize(String f) throws IOException { + long blockSize = super.getBlockSize(f); + FileState fileState = getFileState(f); + if (fileState instanceof CompactFileState) { + blockSize = super.getBlockSize(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath()); + } + return blockSize; + } + + @Override + public void concat(String trg, String [] srcs) throws IOException { + try { + super.concat(trg, srcs); + } catch (IOException e) { + for (String src : srcs) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Concat", "SSM Small File")); + } + } + throw e; + } + } + + @Override + public HdfsFileStatus getFileLinkInfo(String src) throws IOException { + HdfsFileStatus fileStatus = super.getFileLinkInfo(src); + if (fileStatus.getLen() == 0) { + String target = super.getLinkTarget(src); + FileState fileState = getFileState(target); + if (fileState instanceof CompactFileState) { + fileStatus = getFileInfo(target); + } + } + return fileStatus; + } + + @Override + public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) + throws IOException { + MD5MD5CRC32FileChecksum ret = super.getFileChecksum(src, length); + if (ret.getChecksumOpt().getBytesPerChecksum() == 0) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Get file checksum", "SSM Small File")); + } + } + return ret; + } + + @Override + public void setPermission(String src, FsPermission permission) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set permission", "SSM Small File")); + } else { + super.setPermission(src, permission); + } + } + + @Override + public void setOwner(String src, String username, String groupname) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set owner", "SSM Small File")); + } else { + super.setOwner(src, username, groupname); + } + } + + @Override + public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie) + throws IOException { + CorruptFileBlocks corruptFileBlocks = super.listCorruptFileBlocks(path, cookie); + FileState fileState = getFileState(path); + if (fileState instanceof CompactFileState) { + corruptFileBlocks = super.listCorruptFileBlocks(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(), cookie); + } + return corruptFileBlocks; + } + + @Override + public void modifyAclEntries(String src, List aclSpec) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Modify acl entries", "SSM Small File")); + } else { + super.modifyAclEntries(src, aclSpec); + } + } + + @Override + public void removeAclEntries(String src, List aclSpec) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Remove acl entries", "SSM Small File")); + } else { + super.removeAclEntries(src, aclSpec); + } + } + + @Override + public void removeDefaultAcl(String src) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Remove default acl", "SSM Small File")); + } else { + super.removeDefaultAcl(src); + } + } + + @Override + public void removeAcl(String src) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Remove acl", "SSM Small File")); + } else { + super.removeAcl(src); + } + } + + @Override + public void setAcl(String src, List aclSpec) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Set acl", "SSM Small File")); + } else { + super.setAcl(src, aclSpec); + } + } + + @Override + public void createEncryptionZone(String src, String keyName) + throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + throw new IOException(getExceptionMsg("Create encryption zone", "SSM Small File")); + } else { + super.createEncryptionZone(src, keyName); + } + } + + @Override + public void checkAccess(String src, FsAction mode) throws IOException { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + super.checkAccess(((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(), mode); + } else { + super.checkAccess(src, mode); + } + } + + @Override + public boolean isFileClosed(String src) throws IOException { + boolean isFileClosed = super.isFileClosed(src); + if (!isFileClosed) { + FileState fileState = getFileState(src); + if (fileState instanceof CompactFileState) { + String containerFile = ((CompactFileState) fileState) + .getFileContainerInfo().getContainerFilePath(); + isFileClosed = super.isFileClosed(containerFile); + } + } + return isFileClosed; + } + + @Override + public synchronized void close() throws IOException { + try { + super.close(); + } finally { + try { + if (smartClient != null) { + smartClient.close(); + } + } finally { + healthy = false; + } + } + } + + /** + * Report file access event to SSM server. + */ + private void reportFileAccessEvent(String src) { + try { + if (!healthy) { + return; + } + String userName; + try { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } catch (IOException e) { + userName = "Unknown"; + } + smartClient.reportFileAccessEvent(new FileAccessEvent(src, userName)); + } catch (IOException e) { + // Here just ignores that failed to report + LOG.error("Cannot report file access event to SmartServer: " + src + + " , for: " + e.getMessage() + + " , report mechanism will be disabled now in this instance."); + healthy = false; + } + } + + /** + * Check if the smart client is disabled. + */ + private boolean isSmartClientDisabled() { + File idFile = new File(SmartConstants.SMART_CLIENT_DISABLED_ID_FILE); + return idFile.exists(); + } + + /** + * Get the exception message of unsupported operation. + * + * @param operation the hdfs operation name + * @param fileType the type of SSM specify file + * @return the message of unsupported exception + */ + public String getExceptionMsg(String operation, String fileType) { + return String.format("%s is not supported for %s.", operation, fileType); + } + + /** + * Get file state of the specified file. + * + * @param filePath the path of source file + * @return file state of source file + * @throws IOException e + */ + public FileState getFileState(String filePath) throws IOException { + try { + byte[] fileState = getXAttr(filePath, SmartConstants.SMART_FILE_STATE_XATTR_NAME); + if (fileState != null) { + return (FileState) SerializationUtils.deserialize(fileState); + } + } catch (RemoteException e) { + return new NormalFileState(filePath); + } + + return new NormalFileState(filePath); + } +} diff --git a/smart-hadoop-support/smart-hadoop-common/pom.xml b/smart-hadoop-support/smart-hadoop-common/pom.xml new file mode 100644 index 00000000000..495999122b1 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-common/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + + org.smartdata + smart-hadoop-support + 1.5.0-SNAPSHOT + .. + + + smart-hadoop-common + 1.5.0-SNAPSHOT + jar + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-util-ajax + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + test + test-jar + + + diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java similarity index 71% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java rename to smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java index 7f394afc1d7..aad542a1b20 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java @@ -18,25 +18,21 @@ package org.smartdata.hdfs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.SmartInputStreamFactory; import org.apache.hadoop.hdfs.inotify.Event; -import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.*; import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.balancer.KeyManager; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; -import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.security.token.Token; -import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.action.move.StorageGroup; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.OutputStream; +import java.io.*; import java.util.List; public interface CompatibilityHelper { @@ -74,7 +70,22 @@ void replaceBlock( OutputStream getDFSClientAppend(DFSClient client, String dest, int buffersize, long offset) throws IOException; + OutputStream getDFSClientAppend(DFSClient client, String dest, int buffersize) throws IOException; + OutputStream getS3outputStream(String dest, Configuration conf) throws IOException; - SmartInputStreamFactory getSmartInputStreamFactory(); + int getReadTimeOutConstant(); + + Token getAccessToken(KeyManager km, ExtendedBlock eb, StorageGroup target) throws IOException; + + int getIOFileBufferSize(Configuration conf); + + InputStream getVintPrefixed(DataInputStream in) throws IOException; + + LocatedBlocks getLocatedBlocks(HdfsLocatedFileStatus status); + + HdfsFileStatus createHdfsFileStatus( + long length, boolean isdir, int block_replication, long blocksize, long modification_time, + long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, + long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy); } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/CompatibilityHelperLoader.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelperLoader.java similarity index 93% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/CompatibilityHelperLoader.java rename to smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelperLoader.java index c2389cd8ae6..0a125866551 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/CompatibilityHelperLoader.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelperLoader.java @@ -25,6 +25,7 @@ public class CompatibilityHelperLoader { private static CompatibilityHelper instance; private static final String HADOOP_26_HELPER_CLASS = "org.smartdata.hdfs.CompatibilityHelper26"; private static final String HADOOP_27_HELPER_CLASS = "org.smartdata.hdfs.CompatibilityHelper27"; + private static final String HADOOP_31_HELPER_CLASS = "org.smartdata.hdfs.CompatibilityHelper31"; public static final Logger LOG = LoggerFactory.getLogger(CompatibilityHelperLoader.class); @@ -46,8 +47,10 @@ public static CompatibilityHelper getHelper() { Integer second = Integer.parseInt(parts[1]); if (first == 2 && second <= 6) { instance = create(HADOOP_26_HELPER_CLASS); - } else { + } else if (first == 2 && second <= 9){ instance = create(HADOOP_27_HELPER_CLASS); + } else { + instance = create(HADOOP_31_HELPER_CLASS); } } return instance; diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/StorageGroup.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/move/StorageGroup.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/StorageGroup.java rename to smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/move/StorageGroup.java diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterFactory.java b/smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterFactory.java similarity index 85% rename from smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterFactory.java rename to smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterFactory.java index dc18dcfb9b3..c80ecd441da 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterFactory.java +++ b/smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterFactory.java @@ -45,14 +45,21 @@ public static MiniClusterFactory get() clazz = Thread.currentThread() .getContextClassLoader() - .loadClass("org.smartdata.hdfs.MiniClusterFactory27"); - } catch (ClassNotFoundException e) { + .loadClass("org.smartdata.hdfs.MiniClusterFactory31"); + } catch (ClassNotFoundException e1) { try { clazz = Thread.currentThread() .getContextClassLoader() - .loadClass("org.smartdata.hdfs.MiniClusterFactory26"); - } catch (ClassNotFoundException e1) { + .loadClass("org.smartdata.hdfs.MiniClusterFactory27"); + } catch (ClassNotFoundException e2) { + try { + clazz = + Thread.currentThread() + .getContextClassLoader() + .loadClass("org.smartdata.hdfs.MiniClusterFactory26"); + } catch (ClassNotFoundException e3) { + } } } return clazz == null diff --git a/smart-hadoop-support/smart-hadoop/pom.xml b/smart-hadoop-support/smart-hadoop/pom.xml index eedaa5ffb2a..33d21f89fef 100644 --- a/smart-hadoop-support/smart-hadoop/pom.xml +++ b/smart-hadoop-support/smart-hadoop/pom.xml @@ -30,7 +30,79 @@ 1.5.0-SNAPSHOT jar + + + hadoop-2.7 + + true + + + + org.smartdata + smart-hadoop-2 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-2.7 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-2.7 + 1.5.0-SNAPSHOT + test + test-jar + + + + + hadoop-cdh-2.6 + + + org.smartdata + smart-hadoop-2 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-cdh-2.6 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-cdh-2.6 + 1.5.0-SNAPSHOT + test + test-jar + + + + + hadoop-3.1 + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + test + test-jar + + + + + + + org.smartdata + smart-hadoop-common + 1.5.0-SNAPSHOT + org.smartdata smart-common @@ -50,12 +122,32 @@ org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-util-ajax + + - com.squareup tape + + org.smartdata + smart-hadoop-common + 1.5.0-SNAPSHOT + test + test-jar + org.smartdata smart-action @@ -97,5 +189,16 @@ httpclient 4.5.5 + + commons-configuration + commons-configuration + 1.6 + + + commons-beanutils + commons-beanutils + + + diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java index eb2cc65814c..dbbdf2feac4 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/MoverExecutor.java @@ -229,7 +229,7 @@ private void parseSchedulePlan(FileMovePlan plan) throws IOException { .newDatanodeInfo(targetIpAddrs.get(planIndex), targetXferPorts.get(planIndex)); StorageGroup target = new StorageGroup(targetDatanode, targetStorageTypes.get(planIndex)); // generate single move - ReplicaMove replicaMove = new ReplicaMove(block, source, target, nnc, saslClient); + ReplicaMove replicaMove = new ReplicaMove(block, source, target, nnc, saslClient, conf); allMoves.add(replicaMove); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java index 6e3d7e2dfa9..e37923c5180 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/NameNodeConnector.java @@ -82,12 +82,6 @@ public String getBlockpoolID() { return blockpoolID; } - /** @return blocks with locations. */ - public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size) - throws IOException { - return namenode.getBlocks(datanode, size); - } - /** @return live datanode storage reports. */ public DatanodeStorageReport[] getLiveDatanodeStorageReport() throws IOException { diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java index 4c3421f935f..5cd2535ef17 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/move/ReplicaMove.java @@ -17,9 +17,9 @@ */ package org.smartdata.hdfs.action.move; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos; @@ -27,7 +27,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.server.balancer.KeyManager; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.token.Token; @@ -59,15 +58,17 @@ class ReplicaMove { private StorageGroup target; private StorageGroup source; private ReplicaMoveStatus status; + private Configuration conf; - public ReplicaMove(Block block, StorageGroup source, StorageGroup target, - NameNodeConnector nnc, SaslDataTransferClient saslClient) { + public ReplicaMove(Block block, StorageGroup source, StorageGroup target, NameNodeConnector nnc, + SaslDataTransferClient saslClient, Configuration conf) { this.nnc = nnc; this.saslClient = saslClient; this.block = block; this.target = target; this.source = source; this.status = new ReplicaMoveStatus(); + this.conf = conf; } @Override @@ -86,7 +87,7 @@ public void run() { try { sock.connect( NetUtils.createSocketAddr(target.getDatanodeInfo().getXferAddr()), - HdfsServerConstants.READ_TIMEOUT); + CompatibilityHelperLoader.getHelper().getReadTimeOutConstant()); sock.setKeepAlive(true); @@ -94,15 +95,16 @@ public void run() { InputStream unbufIn = sock.getInputStream(); ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(), block); final KeyManager km = nnc.getKeyManager(); - Token accessToken = km.getAccessToken(eb); + Token accessToken = + CompatibilityHelperLoader.getHelper().getAccessToken(km, eb, target); IOStreamPair saslStreams = saslClient.socketSend(sock, unbufOut, unbufIn, km, accessToken, target.getDatanodeInfo()); unbufOut = saslStreams.out; unbufIn = saslStreams.in; out = new DataOutputStream(new BufferedOutputStream(unbufOut, - HdfsConstants.IO_FILE_BUFFER_SIZE)); + CompatibilityHelperLoader.getHelper().getIOFileBufferSize(conf))); in = new DataInputStream(new BufferedInputStream(unbufIn, - HdfsConstants.IO_FILE_BUFFER_SIZE)); + CompatibilityHelperLoader.getHelper().getIOFileBufferSize(conf))); sendRequest(out, eb, accessToken); receiveResponse(in); @@ -140,10 +142,10 @@ private void sendRequest( /** Receive a block copy response from the input stream */ private void receiveResponse(DataInputStream in) throws IOException { DataTransferProtos.BlockOpResponseProto response = - DataTransferProtos.BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in)); + DataTransferProtos.BlockOpResponseProto.parseFrom(CompatibilityHelperLoader.getHelper().getVintPrefixed(in)); while (response.getStatus() == DataTransferProtos.Status.IN_PROGRESS) { // read intermediate responses - response = DataTransferProtos.BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in)); + response = DataTransferProtos.BlockOpResponseProto.parseFrom(CompatibilityHelperLoader.getHelper().getVintPrefixed(in)); } String logInfo = "block move is failed"; checkBlockOpStatus(response, logInfo); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/EventBatchSerializer.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/EventBatchSerializer.java index b561894fc8b..f3b91a2d351 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/EventBatchSerializer.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/EventBatchSerializer.java @@ -30,11 +30,13 @@ import org.apache.hadoop.hdfs.inotify.EventBatch; import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; import org.apache.hadoop.hdfs.protocol.proto.AclProtos; +import org.apache.hadoop.hdfs.protocol.proto.AclProtos.*; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.*; import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto; -import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto; +//import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto; +//import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto; @@ -283,7 +285,7 @@ private static Event.CreateEvent.INodeType createTypeConvert(InotifyProtos.INode } } - public static HdfsProtos.FsPermissionProto convert(FsPermission p) { + public static FsPermissionProto convert(FsPermission p) { return FsPermissionProto.newBuilder().setPerm(p.toExtendedShort()).build(); } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/MovePlanMaker.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/MovePlanMaker.java index 61ebb525726..aa10181919c 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/MovePlanMaker.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/MovePlanMaker.java @@ -149,7 +149,7 @@ private void processFile(String fullPath, HdfsLocatedFileStatus status, String d List types = CompatibilityHelperLoader.getHelper().chooseStorageTypes(policy, status.getReplication()); - final LocatedBlocks locatedBlocks = status.getBlockLocations(); + final LocatedBlocks locatedBlocks = CompatibilityHelperLoader.getHelper().getLocatedBlocks(status); final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete(); List lbs = locatedBlocks.getLocatedBlocks(); for (int i = 0; i < lbs.size(); i++) { diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java index da4bad117e7..110a06fecc1 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java @@ -63,7 +63,6 @@ static void initConf(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); } diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestAllSsdFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestAllSsdFileAction.java similarity index 100% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestAllSsdFileAction.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestAllSsdFileAction.java diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestArchiveFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestArchiveFileAction.java similarity index 100% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestArchiveFileAction.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestArchiveFileAction.java diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestCopy2S3Action.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java similarity index 98% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestCopy2S3Action.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java index bd9f89bc826..2309d6ca482 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestCopy2S3Action.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestCopy2S3Action.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.hdfs.action; +package org.smartdata.hdfs.action.move; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestMoveFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java similarity index 93% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestMoveFileAction.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java index e8d80f4b338..2fab4459f60 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestMoveFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoveFileAction.java @@ -15,13 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.hdfs.action; - -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.DFSTestUtil; +package org.smartdata.hdfs.action.move; + +//import org.apache.hadoop.fs.FSDataOutputStream; +//import org.apache.hadoop.fs.Path; +//import org.apache.hadoop.fs.StorageType; +//import org.apache.hadoop.hdfs.DFSClient; +//import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.hdfs.*; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -33,6 +35,9 @@ import org.smartdata.model.action.FileMovePlan; import org.smartdata.protocol.message.StatusMessage; import org.smartdata.protocol.message.StatusReporter; +import org.smartdata.hdfs.action.AllSsdFileAction; +import org.smartdata.hdfs.action.ArchiveFileAction; +import org.smartdata.hdfs.action.MoveFileAction; import java.io.IOException; import java.net.URI; diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/move/TestMoverExecutor.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoverExecutor.java similarity index 96% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/move/TestMoverExecutor.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoverExecutor.java index 9d78c9ec60a..94c69718764 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/move/TestMoverExecutor.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestMoverExecutor.java @@ -18,10 +18,12 @@ package org.smartdata.hdfs.action.move; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.DFSClient; +//import org.apache.hadoop.fs.FSDataOutputStream; +//import org.apache.hadoop.fs.Path; +//import org.apache.hadoop.fs.StorageType; +//import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.hdfs.*; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestOneSsdFileAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestOneSsdFileAction.java similarity index 100% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestOneSsdFileAction.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestOneSsdFileAction.java diff --git a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestTruncateAction.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java similarity index 88% rename from smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestTruncateAction.java rename to smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java index dc0485682e8..e58d636d5e6 100644 --- a/smart-hadoop-support/smart-hadoop-2.7/src/test/java/org/smartdata/hdfs/action/TestTruncateAction.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/move/TestTruncateAction.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.hdfs.action; +package org.smartdata.hdfs.action.move; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -24,6 +24,7 @@ import org.junit.Test; import org.smartdata.action.MockActionStatusReporter; import org.smartdata.hdfs.MiniClusterHarness; +import org.smartdata.hdfs.action.TruncateAction; import java.io.IOException; import java.util.HashMap; @@ -54,6 +55,10 @@ public void testLocalTruncateFile() throws IOException, InterruptedException { truncateAction.init(args); truncateAction.run(); + // Hadoop 2.6 doesn't support truncate action + if (truncateAction.getActionStatus().getThrowable() instanceof UnsupportedOperationException) { + return; + } Assert.assertTrue(truncateAction.getExpectedAfterRun()); FSDataInputStream in = dfs.open(new Path(srcPath + "/" + file),50); @@ -87,6 +92,10 @@ public void testRemoteTruncateFile() throws IOException, InterruptedException { truncateAction.init(args); truncateAction.run(); + // Hadoop 2.6 doesn't support truncate action + if (truncateAction.getActionStatus().getThrowable() instanceof UnsupportedOperationException) { + return; + } Assert.assertTrue(truncateAction.getExpectedAfterRun()); FSDataInputStream in = dfs.open(new Path(srcPath + "/" + file),50); diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java index 2454e79ae89..b2285f36126 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java @@ -81,7 +81,6 @@ static void initConf(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java index 7c8fbb8e6eb..f9be8e7080c 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyEventApplier.java @@ -28,6 +28,7 @@ import org.junit.Test; import org.mockito.Matchers; import org.mockito.Mockito; +import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.hdfs.HadoopUtil; import org.smartdata.metastore.MetaStore; @@ -73,7 +74,7 @@ public void testApplier() throws Exception { .replication(3) .build(); HdfsFileStatus status1 = - new HdfsFileStatus( + CompatibilityHelperLoader.getHelper().createHdfsFileStatus( 0, false, 1, @@ -191,7 +192,7 @@ public void testApplierCreateEvent() throws Exception { metaStore.insertBackUpInfo(backUpInfo); HdfsFileStatus status1 = - new HdfsFileStatus( + CompatibilityHelperLoader.getHelper().createHdfsFileStatus( 0, false, 2, @@ -308,7 +309,7 @@ private HdfsFileStatus getDummyDirStatus(String file, long fid) { } private HdfsFileStatus doGetDummyStatus(String file, long fid, boolean isdir) { - return new HdfsFileStatus( + return CompatibilityHelperLoader.getHelper().createHdfsFileStatus( 0, isdir, 1, diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyFetcher.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyFetcher.java index 7bee33f38a2..b42349ee24a 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyFetcher.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestInotifyFetcher.java @@ -34,6 +34,7 @@ import org.junit.Test; import org.mockito.Mockito; import org.smartdata.SmartConstants; +import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.hdfs.MiniClusterFactory; import org.smartdata.metastore.MetaStore; import org.smartdata.metastore.TestDaoUtil; @@ -202,5 +203,7 @@ public void run() { } } - public abstract HdfsDataOutputStream append(DFSClient client, String src, int bufferSize) throws IOException; + public HdfsDataOutputStream append(DFSClient client, String src, int bufferSize) throws IOException { + return (HdfsDataOutputStream)CompatibilityHelperLoader.getHelper().getDFSClientAppend(client, src, bufferSize); + } } diff --git a/smart-hadoop-support/smart-inputstream/pom.xml b/smart-hadoop-support/smart-inputstream/pom.xml new file mode 100644 index 00000000000..ad6ba5eee63 --- /dev/null +++ b/smart-hadoop-support/smart-inputstream/pom.xml @@ -0,0 +1,86 @@ + + + + 4.0.0 + + + org.smartdata + smart-hadoop-support + 1.5.0-SNAPSHOT + .. + + + smart-inputstream + 1.5.0-SNAPSHOT + jar + + + + hadoop-2.7 + + true + + + + org.smartdata + smart-hadoop-2 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-2.7 + 1.5.0-SNAPSHOT + + + + + hadoop-cdh-2.6 + + + org.smartdata + smart-hadoop-2 + 1.5.0-SNAPSHOT + + + org.smartdata + smart-hadoop-cdh-2.6 + 1.5.0-SNAPSHOT + + + + + hadoop-3.1 + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + + + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/CompressionInputStream.java b/smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/CompressionInputStream.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/CompressionInputStream.java rename to smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/CompressionInputStream.java diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/S3InputStream.java b/smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/S3InputStream.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/S3InputStream.java rename to smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/S3InputStream.java diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory.java b/smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory.java similarity index 74% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory.java rename to smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory.java index 1fd2947b97e..688e373b4ad 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory.java +++ b/smart-hadoop-support/smart-inputstream/src/main/java/org/apache/hadoop/hdfs/SmartInputStreamFactory.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.fs.UnresolvedLinkException; import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.model.FileState; @@ -25,11 +26,7 @@ /** * Factory to create SmartInputStream with corresponding Hadoop version. */ -public abstract class SmartInputStreamFactory { - public static SmartInputStreamFactory get() { - return CompatibilityHelperLoader.getHelper().getSmartInputStreamFactory(); - } - +public class SmartInputStreamFactory { /** * Get HDFS input stream from dfsClient, file path and its file state. * @@ -40,15 +37,18 @@ public static SmartInputStreamFactory get() { * @return HDFS input stream * @throws IOException if IOException occurs */ - public abstract DFSInputStream create(DFSClient dfsClient, String src, - boolean verifyChecksum, FileState fileState) throws IOException; - - protected DFSInputStream createSmartInputStream(DFSClient dfsClient, String src, - boolean verifyChecksum, FileState fileState) throws IOException{ + public static DFSInputStream create(DFSClient dfsClient, String src, + boolean verifyChecksum, FileState fileState) throws IOException { + dfsClient.checkOpen(); + return createSmartInputStream(dfsClient, src, verifyChecksum, fileState); + } + + protected static DFSInputStream createSmartInputStream(DFSClient dfsClient, String src, + boolean verifyChecksum, FileState fileState) throws IOException { DFSInputStream inputStream; switch (fileState.getFileType()) { case NORMAL: - inputStream = new DFSInputStream(dfsClient, src, verifyChecksum); + inputStream = new SmartInputStream(dfsClient, src, verifyChecksum, fileState); break; case COMPACT: inputStream = new CompactInputStream(dfsClient, verifyChecksum, fileState); diff --git a/smart-integration/pom.xml b/smart-integration/pom.xml index 8239f77207f..bdac0d75853 100644 --- a/smart-integration/pom.xml +++ b/smart-integration/pom.xml @@ -45,7 +45,14 @@ org.smartdata - smart-hadoop-client + smart-hadoop-common + 1.5.0-SNAPSHOT + test + test-jar + + + org.smartdata + smart-inputstream 1.5.0-SNAPSHOT test @@ -139,6 +146,11 @@ test test-jar + + org.mockito + mockito-all + ${mockito.version} + @@ -156,6 +168,12 @@ test test-jar + + org.smartdata + smart-hadoop-client-2.7 + 1.5.0-SNAPSHOT + test + @@ -168,6 +186,36 @@ test test-jar + + org.smartdata + smart-hadoop-client-cdh-2.6 + 1.5.0-SNAPSHOT + test + + + + + hadoop-3.1 + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + test + test-jar + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + test + + + org.smartdata + smart-hadoop-client-3.1 + 1.5.0-SNAPSHOT + test + diff --git a/smart-integration/src/test/java/org/smartdata/integration/cluster/SmartMiniCluster.java b/smart-integration/src/test/java/org/smartdata/integration/cluster/SmartMiniCluster.java index 90d782c6ab7..fae847f5af6 100644 --- a/smart-integration/src/test/java/org/smartdata/integration/cluster/SmartMiniCluster.java +++ b/smart-integration/src/test/java/org/smartdata/integration/cluster/SmartMiniCluster.java @@ -50,7 +50,6 @@ private void initConf(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); } diff --git a/smart-metrics/pom.xml b/smart-metrics/pom.xml index 7e6762c3f79..c1dfbf7fe33 100644 --- a/smart-metrics/pom.xml +++ b/smart-metrics/pom.xml @@ -34,6 +34,20 @@ org.apache.hadoop hadoop-common + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.eclipse.jetty + jetty-webapp + + diff --git a/smart-server/pom.xml b/smart-server/pom.xml index 7579fb51a79..22c6f65f387 100644 --- a/smart-server/pom.xml +++ b/smart-server/pom.xml @@ -51,6 +51,12 @@ org.smartdata smart-engine 1.5.0-SNAPSHOT + + + commons-configuration + commons-configuration + + org.smartdata @@ -66,6 +72,20 @@ org.smartdata smart-zeppelin-server 1.5.0-SNAPSHOT + + + org.codehaus.woodstox + woodstox-core-asl + + + commons-beanutils + commons-beanutils + + + org.eclipse.jetty + jetty-http + + org.slf4j @@ -98,6 +118,19 @@ 1.5.0-SNAPSHOT test test-jar + + + commons-configuration + commons-configuration + + + + + org.smartdata + smart-hadoop-common + 1.5.0-SNAPSHOT + test + test-jar org.apache.hadoop @@ -142,11 +175,6 @@ smart-tidb 1.5.0-SNAPSHOT - - org.smartdata - smart-metastore - 1.5.0-SNAPSHOT - org.smartdata smart-metastore @@ -171,6 +199,11 @@ test test-jar + + org.eclipse.jetty + jetty-http + 9.2.15.v20160210 + @@ -189,6 +222,29 @@ test test-jar + + org.eclipse.jetty + jetty-http + 9.2.15.v20160210 + + + + + hadoop-3.1 + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + test + + + org.smartdata + smart-hadoop-3.1 + 1.5.0-SNAPSHOT + test + test-jar + diff --git a/smart-server/src/test/java/org/smartdata/server/TestSmartServerReConfig.java b/smart-server/src/test/java/org/smartdata/server/TestSmartServerReConfig.java index aa1a6706216..c057b81c62a 100644 --- a/smart-server/src/test/java/org/smartdata/server/TestSmartServerReConfig.java +++ b/smart-server/src/test/java/org/smartdata/server/TestSmartServerReConfig.java @@ -97,7 +97,6 @@ private void initConf(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); } } diff --git a/smart-zeppelin/zeppelin-server/pom.xml b/smart-zeppelin/zeppelin-server/pom.xml index 8b20fe042c2..296924d98ef 100644 --- a/smart-zeppelin/zeppelin-server/pom.xml +++ b/smart-zeppelin/zeppelin-server/pom.xml @@ -271,10 +271,18 @@ 1.5.0-SNAPSHOT + + org.smartdata + smart-hadoop-2 + org.smartdata smart-hadoop-2.7 + + org.smartdata + smart-hadoop-client-2.7 +