From c685309a803d1bca208bc1c2013797cbb776000a Mon Sep 17 00:00:00 2001 From: philo Date: Tue, 4 Sep 2018 09:57:40 +0800 Subject: [PATCH 01/20] support a series of EC actions --- .../java/org/smartdata/SmartConstants.java | 3 +- .../hdfs/action/CheckErasureCodingPolicy.java | 17 +++ .../hdfs/action/ErasureCodingAction.java | 85 +++++++++++ .../hdfs/action/ErasureCodingBase.java | 84 +++++++++++ .../hdfs/action/ListErasureCodingPolicy.java | 31 ++++ .../hdfs/action/UnErasureCodingAction.java | 69 +++++++++ .../smart-hadoop-common/pom.xml | 10 ++ .../org/smartdata/hdfs/action/HdfsAction.java | 0 .../hdfs/action/HdfsActionFactory.java | 11 ++ .../scheduler/ErasureCodingScheduler.java | 138 ++++++++++++++++++ 10 files changed, 447 insertions(+), 1 deletion(-) create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java rename smart-hadoop-support/{smart-hadoop => smart-hadoop-common}/src/main/java/org/smartdata/hdfs/action/HdfsAction.java (100%) create mode 100644 smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java diff --git a/smart-common/src/main/java/org/smartdata/SmartConstants.java b/smart-common/src/main/java/org/smartdata/SmartConstants.java index 9206d43c1f8..04e15e460c1 100644 --- a/smart-common/src/main/java/org/smartdata/SmartConstants.java +++ b/smart-common/src/main/java/org/smartdata/SmartConstants.java @@ -28,7 +28,8 @@ public class SmartConstants { "org.smartdata.hdfs.scheduler.MoverScheduler, " + "org.smartdata.hdfs.scheduler.CopyScheduler, " + "org.smartdata.hdfs.scheduler.Copy2S3Scheduler," - + "org.smartdata.hdfs.scheduler.SmallFileScheduler"; + + "org.smartdata.hdfs.scheduler.SmallFileScheduler" + + "org.smartdata.hdfs.scheduler.ErasureCodingScheduler"; public static final String SMART_HADOOP_LAST_INOTIFY_TXID = "smart_hadoop_last_inotify_txid"; diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java new file mode 100644 index 00000000000..a53504fadd5 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java @@ -0,0 +1,17 @@ +package org.smartdata.hdfs.action; + +import java.util.Map; + +public class CheckErasureCodingPolicy extends HdfsAction { + private String srcPath; + + @Override + public void init(Map args) { + this.srcPath = args.get(HdfsAction.FILE_PATH); + } + + @Override + public void execute() throws Exception { + appendResult(dfsClient.getErasureCodingPolicy(srcPath).toString()); + } +} diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java new file mode 100644 index 00000000000..c5c03aaf79c --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.smartdata.action.ActionException; +import org.smartdata.conf.SmartConf; + +import java.util.HashMap; +import java.util.Map; + +public class ErasureCodingAction extends ErasureCodingBase { + private static final Logger LOG = + LoggerFactory.getLogger(ErasureCodingAction.class); + public static final String EC_POLICY_NAME = "-policy"; + + private String ecPolicyName; + private SmartConf conf; + + @Override + public void init(Map args) { + this.srcPath = args.get(FILE_PATH); + if (args.containsKey(DEST)) { + // It's a temp file kept for converting a file to another with other ec policy + this.destPath = args.get(DEST); + } + if (args.containsKey(EC_POLICY_NAME)) { + this.ecPolicyName = args.get(EC_POLICY_NAME); + } else { + this.conf = getContext().getConf(); + String defaultEcPolicy = conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY, + DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); + this.ecPolicyName = defaultEcPolicy; + } + if (args.containsKey(BUF_SIZE)) { + this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); + } + this.progress = 0.0F; + } + + @Override + protected void execute() throws Exception { + HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); + if (fileStatus == null) { + throw new ActionException("File doesn't exist!"); + } + ValidateEcPolicy(ecPolicyName); + if (fileStatus.getErasureCodingPolicy().getName().equals(ecPolicyName)) { + this.progress = 1.0F; + return; + } + if (fileStatus.isDir()) { + dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); + progress = 1.0F; + return; + } + convert(conf, ecPolicyName); + dfsClient.rename(destPath, srcPath, null); + } + + @Override + public float getProgress() { + return progress; + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java new file mode 100644 index 00000000000..e22c4791f35 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.smartdata.action.ActionException; +import org.smartdata.conf.SmartConf; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +abstract public class ErasureCodingBase extends HdfsAction { + public static final String BUF_SIZE = "-bufSize"; + protected String srcPath; + protected String destPath; + protected int bufferSize = 1024 * 1024; + protected float progress; + public static final String DEST = "-dest"; + + protected void convert(SmartConf conf, String ecPolicyName) throws ActionException { + try { + long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); + DFSInputStream in = dfsClient.open(srcPath, bufferSize, true); + HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); + FsPermission permission = fileStatus.getPermission(); + DFSOutputStream out = + dfsClient.create(destPath, permission, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SHOULD_REPLICATE), + false, (short) 1, blockSize, null, bufferSize, null, null, ecPolicyName); + long bytesRemaining = fileStatus.getLen(); + byte[] buf = new byte[bufferSize]; + while (bytesRemaining > 0L) { + int bytesToRead = + (int) (bytesRemaining < (long) buf.length ? bytesRemaining : + (long) buf.length); + int bytesRead = in.read(buf, 0, bytesToRead); + if (bytesRead == -1) { + break; + } + out.write(buf, 0, bytesRead); + bytesRemaining -= (long) bytesRead; + this.progress = (float) (fileStatus.getLen() - bytesRemaining) / fileStatus.getLen(); + } + } catch (IOException ex) { + throw new ActionException(ex); + } + } + + public void ValidateEcPolicy(String ecPolicyName) throws Exception { + Map ecPolicyNameToState = new HashMap<>(); + for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { + ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); + } + if (!ecPolicyNameToState.keySet().contains(ecPolicyName)) { + throw new ActionException("The given EC policy is not supported!"); + } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED + || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { + throw new ActionException("The given EC policy is not enabled!"); + } + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java new file mode 100644 index 00000000000..05cbde3f326 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import java.util.Map; + +public class ListErasureCodingPolicy extends HdfsAction { + @Override + public void init(Map args) { + } + + @Override + public void execute() throws Exception { + appendResult(dfsClient.getErasureCodingPolicies().toString()); + } +} diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java new file mode 100644 index 00000000000..3d6afa95f06 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; +import org.smartdata.action.ActionException; +import org.smartdata.conf.SmartConf; + +import java.util.Map; + +public class UnErasureCodingAction extends ErasureCodingBase { + private static final String ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME; + private SmartConf conf; + + @Override + public void init(Map args) { + this.conf = getContext().getConf(); + this.srcPath = args.get(FILE_PATH); + if (args.containsKey(DEST)) { + // dest path is under /system/ssm/ec_tmp/file_name_timestamp, the file name is srcName_millisecond + this.destPath = args.get(DEST); + } + if (args.containsKey(BUF_SIZE)) { + this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); + } + this.progress = 0.0F; + } + + @Override + protected void execute() throws Exception { + HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); + if (fileStatus == null) { + throw new ActionException("File doesn't exist!"); + } + ValidateEcPolicy(ecPolicyName); + if (fileStatus.getErasureCodingPolicy().getName().equals(ecPolicyName)) { + this.progress = 1.0F; + return; + } + if (fileStatus.isDir()) { + dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); + progress = 1.0F; + return; + } + convert(conf, ecPolicyName); + dfsClient.rename(destPath, srcPath, null); + } + + @Override + public float getProgress() { + return progress; + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-common/pom.xml b/smart-hadoop-support/smart-hadoop-common/pom.xml index 495999122b1..48d59f2378b 100644 --- a/smart-hadoop-support/smart-hadoop-common/pom.xml +++ b/smart-hadoop-support/smart-hadoop-common/pom.xml @@ -31,6 +31,16 @@ jar + + org.smartdata + smart-common + 1.5.0-SNAPSHOT + + + org.smartdata + smart-action + 1.5.0-SNAPSHOT + org.apache.hadoop hadoop-hdfs diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsAction.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/HdfsAction.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsAction.java rename to smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/action/HdfsAction.java diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java index 3d20ab0cd25..dfe53ec24cc 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java @@ -17,8 +17,11 @@ */ package org.smartdata.hdfs.action; +import org.apache.hadoop.util.VersionInfo; import org.smartdata.action.AbstractActionFactory; +import java.io.IOException; + /** * Built-in smart actions for HDFS system. */ @@ -57,5 +60,13 @@ public class HdfsActionFactory extends AbstractActionFactory { // addAction("diskbalance", DiskBalanceAction.class); // addAction("clusterbalance", ClusterBalanceAction.class); // addAction("setstoragepolicy", SetStoragePolicyAction.class); + String[] parts = VersionInfo.getVersion().split("\\."); + if (Integer.parseInt(parts[0]) == 3) { + addAction(ListErasureCodingPolicy.class); + addAction(CheckErasureCodingPolicy.class); + addAction(ErasureCodingBase.class); + addAction(ErasureCodingAction.class); + addAction(UnErasureCodingAction.class); + } } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java new file mode 100644 index 00000000000..5c68a5df583 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.scheduler; + +//import org.smartdata.hdfs.action.ErasureCodingAction; +import org.apache.hadoop.util.VersionInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.smartdata.SmartContext; +import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.action.HdfsAction; +import org.smartdata.metastore.MetaStore; +import org.smartdata.metastore.MetaStoreException; +import org.smartdata.model.ActionInfo; +import org.smartdata.model.LaunchAction; +import org.smartdata.model.action.ScheduleResult; + +import java.io.IOException; +import java.util.*; + +public class ErasureCodingScheduler extends ActionSchedulerService { + public static final Logger LOG = LoggerFactory.getLogger(ErasureCodingScheduler.class); + private static final List ACTIONS = Arrays.asList("ec", "unec", "checkec"); + // The arguments which can be set by user + public static Set arguments = new HashSet<>(); + public static final String EC_DIR = "/system/ssm/ec_tmp"; + private Set fileLock; + private SmartConf conf; + private MetaStore metaStore; + + public ErasureCodingScheduler(SmartContext context, MetaStore metaStore) { + super(context, metaStore); + this.conf = context.getConf(); + this.metaStore = metaStore; + } + public List getSupportedActions() { + return ACTIONS; + } + + public void init() throws IOException { + fileLock = new HashSet<>(); +// arguments.addAll(Arrays.asList(HdfsAction.FILE_PATH, ErasureCodingAction.EC_POLICY_NAME, +// ErasureCodingAction.BUF_SIZE)); + } + + public void start() throws IOException { + + } + + public void stop() throws IOException { + + } + + @Override + public boolean onSubmit(ActionInfo actionInfo) throws IOException { + if (actionInfo.getArgs().get(HdfsAction.FILE_PATH) == null) { + throw new IOException("No src path is given!"); + } + String[] parts = VersionInfo.getVersion().split("\\."); + if (Integer.parseInt(parts[0]) == 2) { + throw new IOException(actionInfo.getActionName() + + " is not supported on " + VersionInfo.getVersion()); + } +// if (!arguments.containsAll(actionInfo.getArgs().keySet())) { +// LOG.warn("Invalid arguments:" + +// actionInfo.getArgs().keySet().removeAll(arguments)); +// } + return true; + } + + @Override + public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { + if (!ACTIONS.contains(action.getActionType())) { + return ScheduleResult.SUCCESS; + } + String srcPath = action.getArgs().get(HdfsAction.FILE_PATH); + if (srcPath == null) { + actionInfo.appendLog("No file is given in this action!"); + return ScheduleResult.FAIL; + } + if (fileLock.contains(srcPath)) { + return ScheduleResult.FAIL; + } + try { + if (!metaStore.getFile(srcPath).isdir()) { + actionInfo.getArgs().put("-dest", createDest(actionInfo)); + } + } catch (MetaStoreException ex) { + LOG.error("Error occurred for getting file info", ex); + } + + fileLock.add(srcPath); + return ScheduleResult.SUCCESS; + } + + private String createDest(ActionInfo actionInfo) { + // need update to DB + String path = actionInfo.getArgs().get(HdfsAction.FILE_PATH); + String fileName; + int index = path.lastIndexOf("/"); + if (index == path.length() - 1) { + index = path.substring(0, path.length() - 1).indexOf("/"); + fileName = path.substring(index + 1, path.length() - 1); + } else { + fileName = path.substring(index + 1, path.length()); + } + // The dest tmp file is under EC_DIR and + // named by fileName, aidxxx and current time in millisecond with "_" separated + String dest = EC_DIR + "/" + fileName + "_" + "aid" + actionInfo.getActionId() + + "_" + System.currentTimeMillis(); + return dest; + } + + public void postSchedule(ActionInfo actionInfo, ScheduleResult result) { + } + + public void onPreDispatch(LaunchAction action) { + } + + public void onActionFinished(ActionInfo actionInfo) { + fileLock.remove(actionInfo.getArgs().get(HdfsAction.FILE_PATH)); + } +} From 9027db78f3bbda9ce0ad8e463b638613da95e024 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 6 Sep 2018 14:38:39 +0800 Subject: [PATCH 02/20] refine the code --- .../java/org/smartdata/SmartConstants.java | 2 +- smart-hadoop-support/smart-hadoop-3.1/pom.xml | 11 ++- .../hdfs/action/CheckErasureCodingPolicy.java | 41 ++++++++- .../hdfs/action/ErasureCodingAction.java | 39 ++++++--- .../hdfs/action/ErasureCodingBase.java | 39 +++++++-- .../hdfs/action/ListErasureCodingPolicy.java | 25 +++++- .../hdfs/action/UnErasureCodingAction.java | 28 ++++-- .../hdfs/action/TestErasureCodingAction.java | 56 ++++++++++++ .../action/TestUnErasureCodingAction.java | 65 ++++++++++++++ .../smart-hadoop-common/pom.xml | 5 ++ .../java/org/smartdata/hdfs/HadoopUtil.java | 0 .../smartdata/hdfs/MiniClusterHarness.java | 0 .../hdfs/MiniClusterWithStoragesHarness.java | 0 .../hdfs/action/HdfsActionFactory.java | 9 +- .../scheduler/ErasureCodingScheduler.java | 85 ++++++++++++------- 15 files changed, 334 insertions(+), 71 deletions(-) create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java rename smart-hadoop-support/{smart-hadoop => smart-hadoop-common}/src/main/java/org/smartdata/hdfs/HadoopUtil.java (100%) rename smart-hadoop-support/{smart-hadoop => smart-hadoop-common}/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java (100%) rename smart-hadoop-support/{smart-hadoop => smart-hadoop-common}/src/test/java/org/smartdata/hdfs/MiniClusterWithStoragesHarness.java (100%) diff --git a/smart-common/src/main/java/org/smartdata/SmartConstants.java b/smart-common/src/main/java/org/smartdata/SmartConstants.java index 04e15e460c1..70ff96e9ae7 100644 --- a/smart-common/src/main/java/org/smartdata/SmartConstants.java +++ b/smart-common/src/main/java/org/smartdata/SmartConstants.java @@ -28,7 +28,7 @@ public class SmartConstants { "org.smartdata.hdfs.scheduler.MoverScheduler, " + "org.smartdata.hdfs.scheduler.CopyScheduler, " + "org.smartdata.hdfs.scheduler.Copy2S3Scheduler," - + "org.smartdata.hdfs.scheduler.SmallFileScheduler" + + "org.smartdata.hdfs.scheduler.SmallFileScheduler," + "org.smartdata.hdfs.scheduler.ErasureCodingScheduler"; public static final String SMART_HADOOP_LAST_INOTIFY_TXID = diff --git a/smart-hadoop-support/smart-hadoop-3.1/pom.xml b/smart-hadoop-support/smart-hadoop-3.1/pom.xml index a68ac82c0ae..2a37aa8c4cd 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/pom.xml +++ b/smart-hadoop-support/smart-hadoop-3.1/pom.xml @@ -70,6 +70,11 @@ smart-hadoop-common 1.5.0-SNAPSHOT + + org.apache.hadoop + hadoop-aws + ${hadoop.version} + org.apache.hadoop hadoop-hdfs @@ -85,9 +90,9 @@ test-jar - org.apache.hadoop - hadoop-aws - ${hadoop.version} + junit + junit + test diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java index a53504fadd5..44da82da1b0 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java @@ -1,17 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.smartdata.hdfs.action; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.smartdata.action.annotation.ActionSignature; +import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.HadoopUtil; + import java.util.Map; +@ActionSignature( + actionId = "checkec", + displayName = "checkec", + usage = HdfsAction.FILE_PATH + " $src" +) public class CheckErasureCodingPolicy extends HdfsAction { + public static final String RESULT_OF_NULL_EC_POLICY = + "No EC policy is attached!"; + private SmartConf conf; private String srcPath; @Override public void init(Map args) { + super.init(args); + this.conf = getContext().getConf(); this.srcPath = args.get(HdfsAction.FILE_PATH); } @Override public void execute() throws Exception { - appendResult(dfsClient.getErasureCodingPolicy(srcPath).toString()); + this.setDfsClient(HadoopUtil.getDFSClient( + HadoopUtil.getNameNodeUri(conf), conf)); + ErasureCodingPolicy srcEcPolicy = dfsClient.getErasureCodingPolicy(srcPath); + if (srcEcPolicy == null) { + appendResult(RESULT_OF_NULL_EC_POLICY); + } else { + appendResult(srcEcPolicy.toString()); + } } } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index c5c03aaf79c..b69210323e6 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -18,36 +18,43 @@ package org.smartdata.hdfs.action; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; +import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.HadoopUtil; -import java.util.HashMap; import java.util.Map; +@ActionSignature( + actionId = "ec", + displayName = "ec", + usage = HdfsAction.FILE_PATH + " $src " + ErasureCodingAction.EC_POLICY_NAME + " $policy" + + ErasureCodingBase.BUF_SIZE + " $bufSize" +) public class ErasureCodingAction extends ErasureCodingBase { private static final Logger LOG = LoggerFactory.getLogger(ErasureCodingAction.class); public static final String EC_POLICY_NAME = "-policy"; - private String ecPolicyName; private SmartConf conf; + private String ecPolicyName; @Override public void init(Map args) { + super.init(args); + this.conf = getContext().getConf(); this.srcPath = args.get(FILE_PATH); - if (args.containsKey(DEST)) { + if (args.containsKey(TMP_PATH)) { // It's a temp file kept for converting a file to another with other ec policy - this.destPath = args.get(DEST); + this.tmpPath = args.get(TMP_PATH); } if (args.containsKey(EC_POLICY_NAME)) { this.ecPolicyName = args.get(EC_POLICY_NAME); } else { - this.conf = getContext().getConf(); String defaultEcPolicy = conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY, DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); this.ecPolicyName = defaultEcPolicy; @@ -60,22 +67,30 @@ public void init(Map args) { @Override protected void execute() throws Exception { + this.setDfsClient(HadoopUtil.getDFSClient( + HadoopUtil.getNameNodeUri(conf), conf)); + // keep attribute consistent + // HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); if (fileStatus == null) { throw new ActionException("File doesn't exist!"); } ValidateEcPolicy(ecPolicyName); - if (fileStatus.getErasureCodingPolicy().getName().equals(ecPolicyName)) { - this.progress = 1.0F; - return; + ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); + // if the current ecPolicy is already the target one, no need to convert + if (srcEcPolicy != null){ + if (srcEcPolicy.getName().equals(ecPolicyName)) { + this.progress = 1.0F; + return; + } } if (fileStatus.isDir()) { dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); - progress = 1.0F; + this.progress = 1.0F; return; } convert(conf, ecPolicyName); - dfsClient.rename(destPath, srcPath, null); + dfsClient.rename(tmpPath, srcPath, null); } @Override diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index e22c4791f35..1925862d78d 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -36,20 +37,26 @@ abstract public class ErasureCodingBase extends HdfsAction { public static final String BUF_SIZE = "-bufSize"; protected String srcPath; - protected String destPath; + protected String tmpPath; protected int bufferSize = 1024 * 1024; protected float progress; - public static final String DEST = "-dest"; + public static final String TMP_PATH = "-tmp"; protected void convert(SmartConf conf, String ecPolicyName) throws ActionException { + HdfsDataOutputStream outputStream = null; + DFSInputStream in = null; + DFSOutputStream out = null; try { + // append the file to acquire the lock to avoid modifying, no real appending occurs. + outputStream = + dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); - DFSInputStream in = dfsClient.open(srcPath, bufferSize, true); + in = dfsClient.open(srcPath, bufferSize, true); HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); + // use the same FsPermission as srcPath FsPermission permission = fileStatus.getPermission(); - DFSOutputStream out = - dfsClient.create(destPath, permission, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SHOULD_REPLICATE), - false, (short) 1, blockSize, null, bufferSize, null, null, ecPolicyName); + out = dfsClient.create(tmpPath, permission, EnumSet.of(CreateFlag.CREATE), true, + (short) 1, blockSize, null, bufferSize, null, null, ecPolicyName); long bytesRemaining = fileStatus.getLen(); byte[] buf = new byte[bufferSize]; while (bytesRemaining > 0L) { @@ -64,8 +71,22 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti bytesRemaining -= (long) bytesRead; this.progress = (float) (fileStatus.getLen() - bytesRemaining) / fileStatus.getLen(); } - } catch (IOException ex) { + } catch (Exception ex) { throw new ActionException(ex); + } finally { + try { + if (in != null) { + in.close(); + } + if (out != null) { + out.close(); + } + if (outputStream != null) { + outputStream.close(); + } + } catch (IOException ex) { + throw new ActionException(ex); + } } } @@ -75,10 +96,10 @@ public void ValidateEcPolicy(String ecPolicyName) throws Exception { ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); } if (!ecPolicyNameToState.keySet().contains(ecPolicyName)) { - throw new ActionException("The given EC policy is not supported!"); + throw new ActionException("The EC policy " + ecPolicyName + " is not supported!"); } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { - throw new ActionException("The given EC policy is not enabled!"); + throw new ActionException("The EC policy " + ecPolicyName + " is not enabled!"); } } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java index 05cbde3f326..8f88d160630 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java @@ -17,15 +17,38 @@ */ package org.smartdata.hdfs.action; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.smartdata.action.annotation.ActionSignature; +import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.HadoopUtil; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; +@ActionSignature( + actionId = "listec", + displayName = "listec", + usage = "No args" +) public class ListErasureCodingPolicy extends HdfsAction { + private SmartConf conf; + @Override public void init(Map args) { + super.init(args); + this.conf = getContext().getConf(); } @Override public void execute() throws Exception { - appendResult(dfsClient.getErasureCodingPolicies().toString()); + this.setDfsClient(HadoopUtil.getDFSClient( + HadoopUtil.getNameNodeUri(conf), conf)); + List policyInfos = new ArrayList<>(); + for (ErasureCodingPolicyInfo policyInfo : dfsClient.getErasureCodingPolicies()) { + policyInfos.add("{" + policyInfo.toString() + "}"); + } + appendResult(StringUtils.join(policyInfos, ",")); } } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 3d6afa95f06..4a94c29c757 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -17,24 +17,33 @@ */ package org.smartdata.hdfs.action; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.smartdata.action.ActionException; +import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConf; +import org.smartdata.hdfs.HadoopUtil; import java.util.Map; +@ActionSignature( + actionId = "unec", + displayName = "unec", + usage = HdfsAction.FILE_PATH + " $src " + ErasureCodingBase.BUF_SIZE + " $bufSize" +) public class UnErasureCodingAction extends ErasureCodingBase { private static final String ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME; private SmartConf conf; @Override public void init(Map args) { + super.init(args); this.conf = getContext().getConf(); this.srcPath = args.get(FILE_PATH); - if (args.containsKey(DEST)) { + if (args.containsKey(TMP_PATH)) { // dest path is under /system/ssm/ec_tmp/file_name_timestamp, the file name is srcName_millisecond - this.destPath = args.get(DEST); + this.tmpPath = args.get(TMP_PATH); } if (args.containsKey(BUF_SIZE)) { this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); @@ -44,14 +53,21 @@ public void init(Map args) { @Override protected void execute() throws Exception { + // keep attribute consistent + // + this.setDfsClient(HadoopUtil.getDFSClient( + HadoopUtil.getNameNodeUri(conf), conf)); HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); if (fileStatus == null) { throw new ActionException("File doesn't exist!"); } ValidateEcPolicy(ecPolicyName); - if (fileStatus.getErasureCodingPolicy().getName().equals(ecPolicyName)) { - this.progress = 1.0F; - return; + ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); + if (srcEcPolicy != null) { + if (srcEcPolicy.getName().equals(ecPolicyName)) { + this.progress = 1.0F; + return; + } } if (fileStatus.isDir()) { dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); @@ -59,7 +75,7 @@ protected void execute() throws Exception { return; } convert(conf, ecPolicyName); - dfsClient.rename(destPath, srcPath, null); + dfsClient.rename(tmpPath, srcPath, null); } @Override diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java new file mode 100644 index 00000000000..301af816eee --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.junit.Assert; +import org.junit.Test; +import org.smartdata.hdfs.MiniClusterHarness; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class TestErasureCodingAction extends MiniClusterHarness { + + public void createTestFile(String srcPath) throws IOException { + FSDataOutputStream out = dfs.create(new Path(srcPath)); + for (int i = 0; i < 60; i++) { + out.writeByte(1); + } + out.close(); + } + + @Test + public void testExecute() throws Exception { + ErasureCodingAction ecAction = new ErasureCodingAction(); + String srcPath = "/ec/unecfile"; + createTestFile(srcPath); + String destPath = "/ssm/ec_tmp/tmp_file"; + Map args = new HashMap<>(); + args.put(HdfsAction.FILE_PATH, srcPath); + args.put(ErasureCodingBase.TMP_PATH, destPath); + ecAction.init(args); + Assert.assertTrue(ecAction.getExpectedAfterRun()); + Assert.assertEquals(dfsClient.getErasureCodingPolicy(srcPath), + DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); + // compare attribute + } +} \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java new file mode 100644 index 00000000000..b279d14607f --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; +import org.junit.Assert; +import org.junit.Test; +import org.smartdata.hdfs.MiniClusterHarness; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +public class TestUnErasureCodingAction extends MiniClusterHarness { + + public void createTestFile(String srcPath) throws IOException { + int bufferSize = 1024 * 1024; + DFSOutputStream out = + dfsClient.create(srcPath, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, + CreateFlag.OVERWRITE, CreateFlag.SHOULD_REPLICATE), false, (short) 1, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null, bufferSize, null, + null, DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); + for (int i = 0; i < 60; i++) { + out.write(1); + } + out.close(); + } + + @Test + public void testExecute() throws Exception { + UnErasureCodingAction unecAction = new UnErasureCodingAction(); + String srcPath = "/ec/ecfile"; + // create test file with default ecPolicy + createTestFile(srcPath); + String destPath = "/ssm/ec_tmp/tmp_file"; + Map args = new HashMap<>(); + args.put(HdfsAction.FILE_PATH, srcPath); + args.put(ErasureCodingBase.TMP_PATH, destPath); + unecAction.init(args); + Assert.assertTrue(unecAction.getExpectedAfterRun()); + Assert.assertEquals(dfsClient.getErasureCodingPolicy(srcPath), + SystemErasureCodingPolicies.getReplicationPolicy()); + // compare attribute + } +} diff --git a/smart-hadoop-support/smart-hadoop-common/pom.xml b/smart-hadoop-support/smart-hadoop-common/pom.xml index 48d59f2378b..616896860d5 100644 --- a/smart-hadoop-support/smart-hadoop-common/pom.xml +++ b/smart-hadoop-support/smart-hadoop-common/pom.xml @@ -72,5 +72,10 @@ test test-jar + + junit + junit + test + diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HadoopUtil.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/HadoopUtil.java rename to smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java b/smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java rename to smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterHarness.java diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterWithStoragesHarness.java b/smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterWithStoragesHarness.java similarity index 100% rename from smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/MiniClusterWithStoragesHarness.java rename to smart-hadoop-support/smart-hadoop-common/src/test/java/org/smartdata/hdfs/MiniClusterWithStoragesHarness.java diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java index dfe53ec24cc..690b9ef24a8 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java @@ -17,10 +17,8 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.util.VersionInfo; import org.smartdata.action.AbstractActionFactory; - -import java.io.IOException; +import org.smartdata.hdfs.scheduler.ErasureCodingScheduler; /** * Built-in smart actions for HDFS system. @@ -40,7 +38,6 @@ public class HdfsActionFactory extends AbstractActionFactory { addAction(WriteFileAction.class); addAction(CheckStorageAction.class); addAction(SetXAttrAction.class); -// addAction("stripec", StripErasureCodeFileAction.class); // addAction("blockec", BlockErasureCodeFileAction.class); addAction(CopyFileAction.class); addAction(DeleteFileAction.class); @@ -60,11 +57,9 @@ public class HdfsActionFactory extends AbstractActionFactory { // addAction("diskbalance", DiskBalanceAction.class); // addAction("clusterbalance", ClusterBalanceAction.class); // addAction("setstoragepolicy", SetStoragePolicyAction.class); - String[] parts = VersionInfo.getVersion().split("\\."); - if (Integer.parseInt(parts[0]) == 3) { + if (ErasureCodingScheduler.isECSupported()) { addAction(ListErasureCodingPolicy.class); addAction(CheckErasureCodingPolicy.class); - addAction(ErasureCodingBase.class); addAction(ErasureCodingAction.class); addAction(UnErasureCodingAction.class); } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 5c68a5df583..5f38a5290ea 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -17,13 +17,13 @@ */ package org.smartdata.hdfs.scheduler; -//import org.smartdata.hdfs.action.ErasureCodingAction; import org.apache.hadoop.util.VersionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.SmartContext; +import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConf; -import org.smartdata.hdfs.action.HdfsAction; +import org.smartdata.hdfs.action.*; import org.smartdata.metastore.MetaStore; import org.smartdata.metastore.MetaStoreException; import org.smartdata.model.ActionInfo; @@ -31,14 +31,20 @@ import org.smartdata.model.action.ScheduleResult; import java.io.IOException; -import java.util.*; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; public class ErasureCodingScheduler extends ActionSchedulerService { public static final Logger LOG = LoggerFactory.getLogger(ErasureCodingScheduler.class); - private static final List ACTIONS = Arrays.asList("ec", "unec", "checkec"); - // The arguments which can be set by user - public static Set arguments = new HashSet<>(); + private String ecActionID; + private String unecActionID; + private String checkecActionID; + private String listecActionID; + private List actions; public static final String EC_DIR = "/system/ssm/ec_tmp"; + public static final String TMP = "-tmp"; private Set fileLock; private SmartConf conf; private MetaStore metaStore; @@ -47,70 +53,90 @@ public ErasureCodingScheduler(SmartContext context, MetaStore metaStore) { super(context, metaStore); this.conf = context.getConf(); this.metaStore = metaStore; + + if (isECSupported()) { + ecActionID = ErasureCodingAction.class.getAnnotation(ActionSignature.class).actionId(); + unecActionID = UnErasureCodingAction.class.getAnnotation(ActionSignature.class).displayName(); + checkecActionID = CheckErasureCodingPolicy.class.getAnnotation(ActionSignature.class).displayName(); + listecActionID = ListErasureCodingPolicy.class.getAnnotation(ActionSignature.class).displayName(); + actions = Arrays.asList(ecActionID, unecActionID, checkecActionID, listecActionID); + } } public List getSupportedActions() { - return ACTIONS; + return actions; } public void init() throws IOException { fileLock = new HashSet<>(); -// arguments.addAll(Arrays.asList(HdfsAction.FILE_PATH, ErasureCodingAction.EC_POLICY_NAME, -// ErasureCodingAction.BUF_SIZE)); } + @Override public void start() throws IOException { } + @Override public void stop() throws IOException { } @Override public boolean onSubmit(ActionInfo actionInfo) throws IOException { - if (actionInfo.getArgs().get(HdfsAction.FILE_PATH) == null) { - throw new IOException("No src path is given!"); - } - String[] parts = VersionInfo.getVersion().split("\\."); - if (Integer.parseInt(parts[0]) == 2) { + if (!isECSupported()) { throw new IOException(actionInfo.getActionName() + " is not supported on " + VersionInfo.getVersion()); } -// if (!arguments.containsAll(actionInfo.getArgs().keySet())) { -// LOG.warn("Invalid arguments:" + -// actionInfo.getArgs().keySet().removeAll(arguments)); -// } + if (!actionInfo.getActionName().equals(listecActionID)) { + if (actionInfo.getArgs().get(HdfsAction.FILE_PATH) == null) { + throw new IOException("No src path is given!"); + } + } return true; } + public static boolean isECSupported () { + String[] parts = VersionInfo.getVersion().split("\\."); + return Integer.parseInt(parts[0]) == 3; + } + @Override public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { - if (!ACTIONS.contains(action.getActionType())) { + if (!actions.contains(action.getActionType())) { return ScheduleResult.SUCCESS; } + + if (actionInfo.getActionName().equals(listecActionID)) { + return ScheduleResult.SUCCESS; + } + String srcPath = action.getArgs().get(HdfsAction.FILE_PATH); if (srcPath == null) { actionInfo.appendLog("No file is given in this action!"); return ScheduleResult.FAIL; } + + if (actionInfo.getActionName().equals(checkecActionID)) { + return ScheduleResult.SUCCESS; + } + if (fileLock.contains(srcPath)) { return ScheduleResult.FAIL; } try { if (!metaStore.getFile(srcPath).isdir()) { - actionInfo.getArgs().put("-dest", createDest(actionInfo)); + // For ec or unec, add tmp argument + action.getArgs().put(TMP, createTmpPath(action)); } } catch (MetaStoreException ex) { LOG.error("Error occurred for getting file info", ex); } - fileLock.add(srcPath); return ScheduleResult.SUCCESS; } - private String createDest(ActionInfo actionInfo) { + private String createTmpPath(LaunchAction action) { // need update to DB - String path = actionInfo.getArgs().get(HdfsAction.FILE_PATH); + String path = action.getArgs().get(HdfsAction.FILE_PATH); String fileName; int index = path.lastIndexOf("/"); if (index == path.length() - 1) { @@ -121,18 +147,15 @@ private String createDest(ActionInfo actionInfo) { } // The dest tmp file is under EC_DIR and // named by fileName, aidxxx and current time in millisecond with "_" separated - String dest = EC_DIR + "/" + fileName + "_" + "aid" + actionInfo.getActionId() + + String dest = EC_DIR + "/" + fileName + "_" + "aid" + action.getActionId() + "_" + System.currentTimeMillis(); return dest; } - public void postSchedule(ActionInfo actionInfo, ScheduleResult result) { - } - - public void onPreDispatch(LaunchAction action) { - } - public void onActionFinished(ActionInfo actionInfo) { - fileLock.remove(actionInfo.getArgs().get(HdfsAction.FILE_PATH)); + if (actionInfo.getActionName().equals(ecActionID) || + actionInfo.getActionName().equals(unecActionID)) { + fileLock.remove(actionInfo.getArgs().get(HdfsAction.FILE_PATH)); + } } } From b2ed4423db30b216cec5a86c94af13e3a5a27b38 Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 7 Sep 2018 10:40:28 +0800 Subject: [PATCH 03/20] rename original file before rename the ec counterpart --- .../hdfs/action/ErasureCodingAction.java | 22 ++++++++++++++---- .../hdfs/action/ErasureCodingBase.java | 14 ++++++++--- .../hdfs/action/UnErasureCodingAction.java | 23 +++++++++++++++---- .../hdfs/action/TestErasureCodingAction.java | 6 +++-- .../action/TestUnErasureCodingAction.java | 6 +++-- .../scheduler/ErasureCodingScheduler.java | 16 ++++++++----- 6 files changed, 66 insertions(+), 21 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index b69210323e6..9d14b21728e 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -48,9 +48,13 @@ public void init(Map args) { super.init(args); this.conf = getContext().getConf(); this.srcPath = args.get(FILE_PATH); - if (args.containsKey(TMP_PATH)) { - // It's a temp file kept for converting a file to another with other ec policy - this.tmpPath = args.get(TMP_PATH); + if (args.containsKey(EC_TMP)) { + // this is a temp file kept for converting a file to another with other ec policy. + this.ecTmpPath = args.get(EC_TMP); + } + if (args.containsKey(ORIGIN_TMP)) { + // this is a temp file which origin file is named to. + this.originTmpPath = args.get(ORIGIN_TMP); } if (args.containsKey(EC_POLICY_NAME)) { this.ecPolicyName = args.get(EC_POLICY_NAME); @@ -90,7 +94,17 @@ protected void execute() throws Exception { return; } convert(conf, ecPolicyName); - dfsClient.rename(tmpPath, srcPath, null); + dfsClient.rename(srcPath, originTmpPath, null); + dfsClient.rename(ecTmpPath, srcPath, null); + if (!isEquivalence(originTmpPath, srcPath)) { + // keep the original file to the original path if not equivalent + dfsClient.delete(srcPath, false); + dfsClient.rename(originTmpPath, srcPath, null); + LOG.warn("The original file is modified during the conversion."); + throw new ActionException("The original file is modified during the conversion."); + } else { + dfsClient.delete(originTmpPath, false); + } } @Override diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 1925862d78d..6dad0784023 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -37,10 +37,13 @@ abstract public class ErasureCodingBase extends HdfsAction { public static final String BUF_SIZE = "-bufSize"; protected String srcPath; - protected String tmpPath; + protected String ecTmpPath; + protected String originTmpPath; protected int bufferSize = 1024 * 1024; protected float progress; - public static final String TMP_PATH = "-tmp"; + // The values for -ecTmp & -originTmp are assigned by ErasureCodingScheduler. + public static final String EC_TMP = "-ecTmp"; + public static final String ORIGIN_TMP = "-originTmp"; protected void convert(SmartConf conf, String ecPolicyName) throws ActionException { HdfsDataOutputStream outputStream = null; @@ -55,7 +58,7 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); // use the same FsPermission as srcPath FsPermission permission = fileStatus.getPermission(); - out = dfsClient.create(tmpPath, permission, EnumSet.of(CreateFlag.CREATE), true, + out = dfsClient.create(ecTmpPath, permission, EnumSet.of(CreateFlag.CREATE), true, (short) 1, blockSize, null, bufferSize, null, null, ecPolicyName); long bytesRemaining = fileStatus.getLen(); byte[] buf = new byte[bufferSize]; @@ -102,4 +105,9 @@ public void ValidateEcPolicy(String ecPolicyName) throws Exception { throw new ActionException("The EC policy " + ecPolicyName + " is not enabled!"); } } + + public boolean isEquivalence(String path1, String path2) throws IOException { + // to ensure that there is no modification for original file + return dfsClient.getFileInfo(path1).getLen() == dfsClient.getFileInfo(path2).getLen(); + } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 4a94c29c757..1882c261a76 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConf; @@ -33,6 +35,8 @@ usage = HdfsAction.FILE_PATH + " $src " + ErasureCodingBase.BUF_SIZE + " $bufSize" ) public class UnErasureCodingAction extends ErasureCodingBase { + private static final Logger LOG = + LoggerFactory.getLogger(UnErasureCodingAction.class); private static final String ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME; private SmartConf conf; @@ -41,9 +45,11 @@ public void init(Map args) { super.init(args); this.conf = getContext().getConf(); this.srcPath = args.get(FILE_PATH); - if (args.containsKey(TMP_PATH)) { - // dest path is under /system/ssm/ec_tmp/file_name_timestamp, the file name is srcName_millisecond - this.tmpPath = args.get(TMP_PATH); + if (args.containsKey(EC_TMP)) { + this.ecTmpPath = args.get(EC_TMP); + } + if (args.containsKey(ORIGIN_TMP)) { + this.originTmpPath = args.get(ORIGIN_TMP); } if (args.containsKey(BUF_SIZE)) { this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); @@ -75,7 +81,16 @@ protected void execute() throws Exception { return; } convert(conf, ecPolicyName); - dfsClient.rename(tmpPath, srcPath, null); + dfsClient.rename(srcPath, originTmpPath, null); + dfsClient.rename(ecTmpPath, srcPath, null); + if (!isEquivalence(originTmpPath, srcPath)) { + dfsClient.delete(srcPath, false); + dfsClient.rename(originTmpPath, srcPath, null); + LOG.warn("The original file is modified during the conversion."); + throw new ActionException("The original file is modified during the conversion."); + } else { + dfsClient.delete(originTmpPath, false); + } } @Override diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java index 301af816eee..346cb670223 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -43,10 +43,12 @@ public void testExecute() throws Exception { ErasureCodingAction ecAction = new ErasureCodingAction(); String srcPath = "/ec/unecfile"; createTestFile(srcPath); - String destPath = "/ssm/ec_tmp/tmp_file"; + String ecTmpPath = "/ssm/ec_tmp/tmp_file"; + String originTmpPath = "/ssm/origin_tmp/tmp_file"; Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); - args.put(ErasureCodingBase.TMP_PATH, destPath); + args.put(ErasureCodingBase.EC_TMP, ecTmpPath); + args.put(ErasureCodingBase.ORIGIN_TMP, originTmpPath); ecAction.init(args); Assert.assertTrue(ecAction.getExpectedAfterRun()); Assert.assertEquals(dfsClient.getErasureCodingPolicy(srcPath), diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java index b279d14607f..4c08d7afbbc 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -52,10 +52,12 @@ public void testExecute() throws Exception { String srcPath = "/ec/ecfile"; // create test file with default ecPolicy createTestFile(srcPath); - String destPath = "/ssm/ec_tmp/tmp_file"; + String ecTmpPath = "/ssm/ec_tmp/tmp_file"; + String originTmpPath = "/ssm/origin_tmp/tmp_file"; Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); - args.put(ErasureCodingBase.TMP_PATH, destPath); + args.put(ErasureCodingBase.EC_TMP, ecTmpPath); + args.put(ErasureCodingBase.ORIGIN_TMP, originTmpPath); unecAction.init(args); Assert.assertTrue(unecAction.getExpectedAfterRun()); Assert.assertEquals(dfsClient.getErasureCodingPolicy(srcPath), diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 5f38a5290ea..8206f637c98 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -43,8 +43,10 @@ public class ErasureCodingScheduler extends ActionSchedulerService { private String checkecActionID; private String listecActionID; private List actions; - public static final String EC_DIR = "/system/ssm/ec_tmp"; - public static final String TMP = "-tmp"; + public static final String EC_DIR = "/system/ssm/ec_tmp/"; + public static final String ORIGIN_DIR = "/system/ssm/origin_tmp/"; + public static final String EC_TMP = "-ecTmp"; + public static final String ORIGIN_TMP = "-originTmp"; private Set fileLock; private SmartConf conf; private MetaStore metaStore; @@ -125,7 +127,9 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { try { if (!metaStore.getFile(srcPath).isdir()) { // For ec or unec, add tmp argument - action.getArgs().put(TMP, createTmpPath(action)); + String tmpName = createTmpName(action); + action.getArgs().put(EC_TMP, EC_DIR + tmpName); + actionInfo.getArgs().put(ORIGIN_TMP, ORIGIN_DIR + tmpName); } } catch (MetaStoreException ex) { LOG.error("Error occurred for getting file info", ex); @@ -134,7 +138,7 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { return ScheduleResult.SUCCESS; } - private String createTmpPath(LaunchAction action) { + private String createTmpName(LaunchAction action) { // need update to DB String path = action.getArgs().get(HdfsAction.FILE_PATH); String fileName; @@ -147,9 +151,9 @@ private String createTmpPath(LaunchAction action) { } // The dest tmp file is under EC_DIR and // named by fileName, aidxxx and current time in millisecond with "_" separated - String dest = EC_DIR + "/" + fileName + "_" + "aid" + action.getActionId() + + String tmpName = fileName + "_" + "aid" + action.getActionId() + "_" + System.currentTimeMillis(); - return dest; + return tmpName; } public void onActionFinished(ActionInfo actionInfo) { From aa454420b9fff77f70804a5feb4d7c5c77ac0c8a Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 7 Sep 2018 13:48:58 +0800 Subject: [PATCH 04/20] rename with overwrite --- .../hdfs/action/ErasureCodingAction.java | 15 --------------- .../smartdata/hdfs/action/ErasureCodingBase.java | 3 ++- .../hdfs/action/UnErasureCodingAction.java | 13 ------------- .../hdfs/scheduler/ErasureCodingScheduler.java | 6 ++---- 4 files changed, 4 insertions(+), 33 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 9d14b21728e..4f014292b29 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -52,10 +52,6 @@ public void init(Map args) { // this is a temp file kept for converting a file to another with other ec policy. this.ecTmpPath = args.get(EC_TMP); } - if (args.containsKey(ORIGIN_TMP)) { - // this is a temp file which origin file is named to. - this.originTmpPath = args.get(ORIGIN_TMP); - } if (args.containsKey(EC_POLICY_NAME)) { this.ecPolicyName = args.get(EC_POLICY_NAME); } else { @@ -94,17 +90,6 @@ protected void execute() throws Exception { return; } convert(conf, ecPolicyName); - dfsClient.rename(srcPath, originTmpPath, null); - dfsClient.rename(ecTmpPath, srcPath, null); - if (!isEquivalence(originTmpPath, srcPath)) { - // keep the original file to the original path if not equivalent - dfsClient.delete(srcPath, false); - dfsClient.rename(originTmpPath, srcPath, null); - LOG.warn("The original file is modified during the conversion."); - throw new ActionException("The original file is modified during the conversion."); - } else { - dfsClient.delete(originTmpPath, false); - } } @Override diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 6dad0784023..d1c87d5439f 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -18,6 +18,7 @@ package org.smartdata.hdfs.action; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSInputStream; @@ -38,7 +39,6 @@ abstract public class ErasureCodingBase extends HdfsAction { public static final String BUF_SIZE = "-bufSize"; protected String srcPath; protected String ecTmpPath; - protected String originTmpPath; protected int bufferSize = 1024 * 1024; protected float progress; // The values for -ecTmp & -originTmp are assigned by ErasureCodingScheduler. @@ -74,6 +74,7 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti bytesRemaining -= (long) bytesRead; this.progress = (float) (fileStatus.getLen() - bytesRemaining) / fileStatus.getLen(); } + dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); } catch (Exception ex) { throw new ActionException(ex); } finally { diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 1882c261a76..d407aa6067a 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -48,9 +48,6 @@ public void init(Map args) { if (args.containsKey(EC_TMP)) { this.ecTmpPath = args.get(EC_TMP); } - if (args.containsKey(ORIGIN_TMP)) { - this.originTmpPath = args.get(ORIGIN_TMP); - } if (args.containsKey(BUF_SIZE)) { this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); } @@ -81,16 +78,6 @@ protected void execute() throws Exception { return; } convert(conf, ecPolicyName); - dfsClient.rename(srcPath, originTmpPath, null); - dfsClient.rename(ecTmpPath, srcPath, null); - if (!isEquivalence(originTmpPath, srcPath)) { - dfsClient.delete(srcPath, false); - dfsClient.rename(originTmpPath, srcPath, null); - LOG.warn("The original file is modified during the conversion."); - throw new ActionException("The original file is modified during the conversion."); - } else { - dfsClient.delete(originTmpPath, false); - } } @Override diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 8206f637c98..eed5c1e98f3 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -44,9 +44,7 @@ public class ErasureCodingScheduler extends ActionSchedulerService { private String listecActionID; private List actions; public static final String EC_DIR = "/system/ssm/ec_tmp/"; - public static final String ORIGIN_DIR = "/system/ssm/origin_tmp/"; public static final String EC_TMP = "-ecTmp"; - public static final String ORIGIN_TMP = "-originTmp"; private Set fileLock; private SmartConf conf; private MetaStore metaStore; @@ -126,10 +124,10 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { } try { if (!metaStore.getFile(srcPath).isdir()) { - // For ec or unec, add tmp argument + // For ec or unec, add ecTmp argument String tmpName = createTmpName(action); action.getArgs().put(EC_TMP, EC_DIR + tmpName); - actionInfo.getArgs().put(ORIGIN_TMP, ORIGIN_DIR + tmpName); + actionInfo.getArgs().put(EC_TMP, EC_DIR + tmpName); } } catch (MetaStoreException ex) { LOG.error("Error occurred for getting file info", ex); From ec616cbc86d3ad878f74e2414ff0ce124b50448b Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 7 Sep 2018 15:16:02 +0800 Subject: [PATCH 05/20] modify unec action --- .../hdfs/action/ErasureCodingAction.java | 16 ++++++++++++++ .../hdfs/action/ErasureCodingBase.java | 22 ------------------- .../hdfs/action/UnErasureCodingAction.java | 14 ++++++------ 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 4f014292b29..51a09725af9 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -19,6 +19,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +29,7 @@ import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; +import java.util.HashMap; import java.util.Map; @ActionSignature( @@ -92,6 +95,19 @@ protected void execute() throws Exception { convert(conf, ecPolicyName); } + public void ValidateEcPolicy(String ecPolicyName) throws Exception { + Map ecPolicyNameToState = new HashMap<>(); + for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { + ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); + } + if (!ecPolicyNameToState.keySet().contains(ecPolicyName)) { + throw new ActionException("The EC policy " + ecPolicyName + " is not supported!"); + } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED + || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { + throw new ActionException("The EC policy " + ecPolicyName + " is not enabled!"); + } + } + @Override public float getProgress() { return progress; diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index d1c87d5439f..c5a7bd5bd01 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -24,16 +24,12 @@ import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.smartdata.action.ActionException; import org.smartdata.conf.SmartConf; import java.io.IOException; import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; abstract public class ErasureCodingBase extends HdfsAction { public static final String BUF_SIZE = "-bufSize"; @@ -93,22 +89,4 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti } } } - - public void ValidateEcPolicy(String ecPolicyName) throws Exception { - Map ecPolicyNameToState = new HashMap<>(); - for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { - ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); - } - if (!ecPolicyNameToState.keySet().contains(ecPolicyName)) { - throw new ActionException("The EC policy " + ecPolicyName + " is not supported!"); - } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED - || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { - throw new ActionException("The EC policy " + ecPolicyName + " is not enabled!"); - } - } - - public boolean isEquivalence(String path1, String path2) throws IOException { - // to ensure that there is no modification for original file - return dfsClient.getFileInfo(path1).getLen() == dfsClient.getFileInfo(path2).getLen(); - } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index d407aa6067a..465c2276097 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,8 @@ public class UnErasureCodingAction extends ErasureCodingBase { private static final Logger LOG = LoggerFactory.getLogger(UnErasureCodingAction.class); - private static final String ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME; + private static final String ecPolicyName = + SystemErasureCodingPolicies.getReplicationPolicy().getName(); private SmartConf conf; @Override @@ -64,13 +66,11 @@ protected void execute() throws Exception { if (fileStatus == null) { throw new ActionException("File doesn't exist!"); } - ValidateEcPolicy(ecPolicyName); ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); - if (srcEcPolicy != null) { - if (srcEcPolicy.getName().equals(ecPolicyName)) { - this.progress = 1.0F; - return; - } + // if ecPolicy is null, it means replication. + if (srcEcPolicy == null) { + this.progress = 1.0F; + return; } if (fileStatus.isDir()) { dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); From 333972ce1a11559b55029da6472f9dec7f45d023 Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 7 Sep 2018 15:37:11 +0800 Subject: [PATCH 06/20] get lease by appending way only for ec action --- .../hdfs/action/ErasureCodingAction.java | 19 ++++++++++++++++++- .../hdfs/action/ErasureCodingBase.java | 10 ---------- .../hdfs/action/UnErasureCodingAction.java | 3 ++- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 51a09725af9..691306ad3b6 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -17,7 +17,10 @@ */ package org.smartdata.hdfs.action; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.Options; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; @@ -29,6 +32,7 @@ import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -92,7 +96,20 @@ protected void execute() throws Exception { this.progress = 1.0F; return; } - convert(conf, ecPolicyName); + HdfsDataOutputStream outputStream = null; + try { + // append the file to acquire the lock to avoid modifying, no real appending occurs. + outputStream = + dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); + convert(conf, ecPolicyName); + dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); + } catch (ActionException ex) { + throw new ActionException(ex); + } finally { + if (outputStream != null) { + outputStream.close(); + } + } } public void ValidateEcPolicy(String ecPolicyName) throws Exception { diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index c5a7bd5bd01..41ece69735f 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -18,12 +18,10 @@ package org.smartdata.hdfs.action; import org.apache.hadoop.fs.CreateFlag; -import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSOutputStream; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.smartdata.action.ActionException; import org.smartdata.conf.SmartConf; @@ -42,13 +40,9 @@ abstract public class ErasureCodingBase extends HdfsAction { public static final String ORIGIN_TMP = "-originTmp"; protected void convert(SmartConf conf, String ecPolicyName) throws ActionException { - HdfsDataOutputStream outputStream = null; DFSInputStream in = null; DFSOutputStream out = null; try { - // append the file to acquire the lock to avoid modifying, no real appending occurs. - outputStream = - dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); in = dfsClient.open(srcPath, bufferSize, true); HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); @@ -70,7 +64,6 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti bytesRemaining -= (long) bytesRead; this.progress = (float) (fileStatus.getLen() - bytesRemaining) / fileStatus.getLen(); } - dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); } catch (Exception ex) { throw new ActionException(ex); } finally { @@ -81,9 +74,6 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti if (out != null) { out.close(); } - if (outputStream != null) { - outputStream.close(); - } } catch (IOException ex) { throw new ActionException(ex); } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 465c2276097..f18a2ff6c01 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -17,10 +17,10 @@ */ package org.smartdata.hdfs.action; +import org.apache.hadoop.fs.Options; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; @@ -78,6 +78,7 @@ protected void execute() throws Exception { return; } convert(conf, ecPolicyName); + dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); } @Override From cd0888d16eddf5743093cd029be0f65da9cdbb37 Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 10 Sep 2018 10:36:37 +0800 Subject: [PATCH 07/20] refine the code --- .../hdfs/action/CheckErasureCodingPolicy.java | 2 +- .../hdfs/action/ErasureCodingAction.java | 35 ++++++++-- .../hdfs/action/ErasureCodingBase.java | 4 +- .../hdfs/action/UnErasureCodingAction.java | 26 ++++++-- .../hdfs/action/TestErasureCodingAction.java | 57 +++++++++------- .../action/TestErasureCodingActionBase.java | 45 +++++++++++++ .../action/TestUnErasureCodingAction.java | 65 ++++++++++--------- 7 files changed, 170 insertions(+), 64 deletions(-) create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java index 44da82da1b0..cfabd5bb1c4 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java @@ -31,7 +31,7 @@ ) public class CheckErasureCodingPolicy extends HdfsAction { public static final String RESULT_OF_NULL_EC_POLICY = - "No EC policy is attached!"; + "The EC policy is replication."; private SmartConf conf; private String srcPath; diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 691306ad3b6..49254a3674d 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -74,6 +74,13 @@ public void init(Map args) { @Override protected void execute() throws Exception { + final String MATCH_RESULT = + "The current EC policy is matched with the target one."; + final String DIR_RESULT = + "The EC policy is set successfully for the given directory."; + final String CONVERT_RESULT = + "The file is converted successfully with the given or default ec policy."; + this.setDfsClient(HadoopUtil.getDFSClient( HadoopUtil.getNameNodeUri(conf), conf)); // keep attribute consistent @@ -87,6 +94,14 @@ protected void execute() throws Exception { // if the current ecPolicy is already the target one, no need to convert if (srcEcPolicy != null){ if (srcEcPolicy.getName().equals(ecPolicyName)) { + appendResult(MATCH_RESULT); + this.progress = 1.0F; + return; + } + } else { + // if ecPolicy is null, it means replication. + if (ecPolicyName.equals(REPLICATION_POLICY_NAME)) { + appendResult(MATCH_RESULT); this.progress = 1.0F; return; } @@ -94,16 +109,28 @@ protected void execute() throws Exception { if (fileStatus.isDir()) { dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); this.progress = 1.0F; + appendResult(DIR_RESULT); return; } HdfsDataOutputStream outputStream = null; try { - // append the file to acquire the lock to avoid modifying, no real appending occurs. - outputStream = - dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); + // a file only with replication policy can be appended. + if (srcEcPolicy.equals(REPLICATION_POLICY_NAME)) { + // append the file to acquire the lock to avoid modifying, no real appending occurs. + outputStream = + dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); + } convert(conf, ecPolicyName); dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); + appendResult(CONVERT_RESULT); + if (srcEcPolicy == null) { + appendResult("The previous EC policy is replication."); + } else { + appendResult(String.format("The previous EC policy is {}.", srcEcPolicy.getName())); + } + appendResult(String.format("The current EC policy is {}.", ecPolicyName)); } catch (ActionException ex) { + // delete tmp file throw new ActionException(ex); } finally { if (outputStream != null) { @@ -117,7 +144,7 @@ public void ValidateEcPolicy(String ecPolicyName) throws Exception { for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); } - if (!ecPolicyNameToState.keySet().contains(ecPolicyName)) { + if (!ecPolicyNameToState.keySet().contains(ecPolicyName) || !ecPolicyName.equals(REPLICATION_POLICY_NAME)) { throw new ActionException("The EC policy " + ecPolicyName + " is not supported!"); } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 41ece69735f..5395dd9fad0 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.smartdata.action.ActionException; import org.smartdata.conf.SmartConf; @@ -37,7 +38,8 @@ abstract public class ErasureCodingBase extends HdfsAction { protected float progress; // The values for -ecTmp & -originTmp are assigned by ErasureCodingScheduler. public static final String EC_TMP = "-ecTmp"; - public static final String ORIGIN_TMP = "-originTmp"; + public static final String REPLICATION_POLICY_NAME = + SystemErasureCodingPolicies.getReplicationPolicy().getName(); protected void convert(SmartConf conf, String ecPolicyName) throws ActionException { DFSInputStream in = null; diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index f18a2ff6c01..33344532196 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -20,7 +20,6 @@ import org.apache.hadoop.fs.Options; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; @@ -38,8 +37,7 @@ public class UnErasureCodingAction extends ErasureCodingBase { private static final Logger LOG = LoggerFactory.getLogger(UnErasureCodingAction.class); - private static final String ecPolicyName = - SystemErasureCodingPolicies.getReplicationPolicy().getName(); + private String ecPolicyName; private SmartConf conf; @Override @@ -47,6 +45,7 @@ public void init(Map args) { super.init(args); this.conf = getContext().getConf(); this.srcPath = args.get(FILE_PATH); + this.ecPolicyName = REPLICATION_POLICY_NAME; if (args.containsKey(EC_TMP)) { this.ecTmpPath = args.get(EC_TMP); } @@ -58,6 +57,13 @@ public void init(Map args) { @Override protected void execute() throws Exception { + final String MATCH_RESULT = + "The current EC policy is replication already."; + final String DIR_RESULT = + "The replication EC policy is set successfully for the given directory."; + final String CONVERT_RESULT = + "The file is converted successfully with replication EC policy."; + // keep attribute consistent // this.setDfsClient(HadoopUtil.getDFSClient( @@ -70,15 +76,25 @@ protected void execute() throws Exception { // if ecPolicy is null, it means replication. if (srcEcPolicy == null) { this.progress = 1.0F; + appendResult(MATCH_RESULT); return; } if (fileStatus.isDir()) { dfsClient.setErasureCodingPolicy(srcPath, ecPolicyName); progress = 1.0F; + appendResult(DIR_RESULT); return; } - convert(conf, ecPolicyName); - dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); + try { + convert(conf, ecPolicyName); + dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); + appendResult(CONVERT_RESULT); + appendResult(String.format("The previous EC policy is {}.", srcEcPolicy.getName())); + appendResult(String.format("The current EC policy is {}.", REPLICATION_POLICY_NAME)); + } catch (ActionException ex) { + // delete tmp file + throw new ActionException(ex); + } } @Override diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java index 346cb670223..d600413e59b 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -17,42 +17,55 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.junit.Assert; +import static org.junit.Assert.*; import org.junit.Test; -import org.smartdata.hdfs.MiniClusterHarness; -import java.io.IOException; import java.util.HashMap; import java.util.Map; -public class TestErasureCodingAction extends MiniClusterHarness { - - public void createTestFile(String srcPath) throws IOException { - FSDataOutputStream out = dfs.create(new Path(srcPath)); - for (int i = 0; i < 60; i++) { - out.writeByte(1); - } - out.close(); - } +public class TestErasureCodingAction extends TestErasureCodingActionBase { @Test - public void testExecute() throws Exception { + public void testEcActionForFile() throws Exception { + String srcPath = "/ec/test_file"; + createTestFile(srcPath, 1000); + // the file is stored in replication + assertEquals(null, dfsClient.getErasureCodingPolicy(srcPath)); + ErasureCodingAction ecAction = new ErasureCodingAction(); - String srcPath = "/ec/unecfile"; - createTestFile(srcPath); String ecTmpPath = "/ssm/ec_tmp/tmp_file"; - String originTmpPath = "/ssm/origin_tmp/tmp_file"; Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); - args.put(ErasureCodingBase.ORIGIN_TMP, originTmpPath); + args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); + ecAction.init(args); + ecAction.execute(); + assertTrue(ecAction.getExpectedAfterRun()); + // the file is stored in ec with default policy + assertEquals(dfsClient.getErasureCodingPolicy(srcPath), ecPolicy); + // compare attribute + } + + @Test + public void testEcActionForDir() throws Exception { + String srcDirPath = "/test_dir/"; + dfs.mkdirs(new Path(srcDirPath)); + assertEquals(null, dfsClient.getErasureCodingPolicy(srcDirPath)); + + ErasureCodingAction ecAction = new ErasureCodingAction(); + Map args = new HashMap<>(); + args.put(HdfsAction.FILE_PATH, srcDirPath); + args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); ecAction.init(args); - Assert.assertTrue(ecAction.getExpectedAfterRun()); - Assert.assertEquals(dfsClient.getErasureCodingPolicy(srcPath), - DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); + ecAction.execute(); + assertTrue(ecAction.getExpectedAfterRun()); + assertEquals(dfsClient.getErasureCodingPolicy(srcDirPath), ecPolicy); + + String srcFilePath = "/test_dir/test_file"; + createTestFile(srcFilePath, 1000); + // The newly created file should has the same EC policy as parent directory. + assertEquals(dfsClient.getErasureCodingPolicy(srcFilePath), ecPolicy); // compare attribute } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java new file mode 100644 index 00000000000..94915d0aba6 --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; +import org.junit.Before; +import org.smartdata.hdfs.MiniClusterHarness; + +import java.io.IOException; + +public class TestErasureCodingActionBase extends MiniClusterHarness { + ErasureCodingPolicy ecPolicy; + + @Before + public void init() throws Exception { + // use ErasureCodeConstants.XOR_2_1_SCHEMA + ecPolicy = SystemErasureCodingPolicies.getPolicies().get(3); + cluster = new MiniDFSCluster.Builder(smartContext.getConf()). + numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()). + build(); + } + + public void createTestFile(String srcPath, long length) throws IOException { + DFSTestUtil.createFile(dfs, new Path(srcPath), length, (short) 3, 0L); + } +} diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java index 4c08d7afbbc..ad79fd96136 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -17,51 +17,54 @@ */ package org.smartdata.hdfs.action; -import org.apache.hadoop.fs.CreateFlag; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSOutputStream; -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.junit.Assert; +import org.apache.hadoop.fs.Path; +import static org.junit.Assert.*; import org.junit.Test; -import org.smartdata.hdfs.MiniClusterHarness; -import java.io.IOException; -import java.util.EnumSet; import java.util.HashMap; import java.util.Map; -public class TestUnErasureCodingAction extends MiniClusterHarness { - - public void createTestFile(String srcPath) throws IOException { - int bufferSize = 1024 * 1024; - DFSOutputStream out = - dfsClient.create(srcPath, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, - CreateFlag.OVERWRITE, CreateFlag.SHOULD_REPLICATE), false, (short) 1, - DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null, bufferSize, null, - null, DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); - for (int i = 0; i < 60; i++) { - out.write(1); - } - out.close(); - } +public class TestUnErasureCodingAction extends TestErasureCodingActionBase { @Test public void testExecute() throws Exception { + String testDir = "/test_dir"; + dfs.mkdirs(new Path(testDir)); + dfs.setErasureCodingPolicy(new Path(testDir), ecPolicy.getName()); + // create test file, its EC policy should be consistent with parent dir, i.e., ecPolicy + String srcPath = testDir + "/ec_file"; + createTestFile(srcPath, 1000); + assertEquals(dfsClient.getErasureCodingPolicy(srcPath), ecPolicy); + UnErasureCodingAction unecAction = new UnErasureCodingAction(); - String srcPath = "/ec/ecfile"; - // create test file with default ecPolicy - createTestFile(srcPath); String ecTmpPath = "/ssm/ec_tmp/tmp_file"; - String originTmpPath = "/ssm/origin_tmp/tmp_file"; Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); - args.put(ErasureCodingBase.ORIGIN_TMP, originTmpPath); unecAction.init(args); - Assert.assertTrue(unecAction.getExpectedAfterRun()); - Assert.assertEquals(dfsClient.getErasureCodingPolicy(srcPath), - SystemErasureCodingPolicies.getReplicationPolicy()); + unecAction.execute(); + assertTrue(unecAction.getExpectedAfterRun()); + assertNull(dfsClient.getErasureCodingPolicy(srcPath)); // compare attribute } + + @Test + public void testEcActionForDir() throws Exception { + String testDir = "/test_dir"; + dfs.mkdirs(new Path(testDir)); + dfs.setErasureCodingPolicy(new Path(testDir), ecPolicy.getName()); + assertEquals(dfsClient.getErasureCodingPolicy(testDir), ecPolicy); + + UnErasureCodingAction unecAction = new UnErasureCodingAction(); + Map args = new HashMap<>(); + args.put(HdfsAction.FILE_PATH, testDir); + unecAction.init(args); + unecAction.execute(); + assertNull(dfs.getErasureCodingPolicy(new Path(testDir))); + + // create test file, its EC policy is expected to be replication + String srcPath = testDir + "/ec_file"; + createTestFile(srcPath, 1000); + assertNull(dfs.getErasureCodingPolicy(new Path(srcPath))); + } } From 705b64752c640198e9989ef985e8831dc6905704 Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 10 Sep 2018 11:28:36 +0800 Subject: [PATCH 08/20] keep attributes consistent --- .../hdfs/action/ErasureCodingAction.java | 10 ++++++---- .../hdfs/action/ErasureCodingBase.java | 17 +++++++++++++++++ .../hdfs/action/UnErasureCodingAction.java | 1 + .../hdfs/action/TestErasureCodingAction.java | 2 -- .../hdfs/action/TestUnErasureCodingAction.java | 1 - 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 49254a3674d..1f4903c8ab5 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -32,6 +32,7 @@ import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; +import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -80,16 +81,16 @@ protected void execute() throws Exception { "The EC policy is set successfully for the given directory."; final String CONVERT_RESULT = "The file is converted successfully with the given or default ec policy."; - + this.setDfsClient(HadoopUtil.getDFSClient( HadoopUtil.getNameNodeUri(conf), conf)); // keep attribute consistent - // + HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); if (fileStatus == null) { throw new ActionException("File doesn't exist!"); } - ValidateEcPolicy(ecPolicyName); + validateEcPolicy(ecPolicyName); ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); // if the current ecPolicy is already the target one, no need to convert if (srcEcPolicy != null){ @@ -121,6 +122,7 @@ protected void execute() throws Exception { dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); } convert(conf, ecPolicyName); + setAttributes(srcPath, ecTmpPath); dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendResult(CONVERT_RESULT); if (srcEcPolicy == null) { @@ -139,7 +141,7 @@ protected void execute() throws Exception { } } - public void ValidateEcPolicy(String ecPolicyName) throws Exception { + public void validateEcPolicy(String ecPolicyName) throws Exception { Map ecPolicyNameToState = new HashMap<>(); for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 5395dd9fad0..f965f3f3a39 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -18,6 +18,7 @@ package org.smartdata.hdfs.action; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSInputStream; @@ -29,6 +30,7 @@ import java.io.IOException; import java.util.EnumSet; +import java.util.Map; abstract public class ErasureCodingBase extends HdfsAction { public static final String BUF_SIZE = "-bufSize"; @@ -81,4 +83,19 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti } } } + + // set attributes for dest to keep them consistent with their counterpart of src + protected void setAttributes(String src, String dest) throws IOException { + HdfsFileStatus fileStatus = dfsClient.getFileInfo(src); + dfsClient.setOwner(dest, fileStatus.getOwner(), fileStatus.getGroup()); + dfsClient.setPermission(dest, fileStatus.getPermission()); + dfsClient.setStoragePolicy(dest, dfsClient.getStoragePolicy(src).getName()); + // check whether mtime is changed after rename + dfsClient.setTimes(dest, fileStatus.getModificationTime(), fileStatus.getAccessTime()); + dfsClient.setAcl(dest, dfsClient.getAclStatus(src).getEntries()); + for(Map.Entry entry : dfsClient.getXAttrs(src).entrySet()) { + dfsClient.setXAttr(dest, entry.getKey(), entry.getValue(), + EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); + } + } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 33344532196..19f461d3a83 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -87,6 +87,7 @@ protected void execute() throws Exception { } try { convert(conf, ecPolicyName); + setAttributes(srcPath, ecTmpPath); dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendResult(CONVERT_RESULT); appendResult(String.format("The previous EC policy is {}.", srcEcPolicy.getName())); diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java index d600413e59b..2189abfcfe3 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -44,7 +44,6 @@ public void testEcActionForFile() throws Exception { assertTrue(ecAction.getExpectedAfterRun()); // the file is stored in ec with default policy assertEquals(dfsClient.getErasureCodingPolicy(srcPath), ecPolicy); - // compare attribute } @Test @@ -66,6 +65,5 @@ public void testEcActionForDir() throws Exception { createTestFile(srcFilePath, 1000); // The newly created file should has the same EC policy as parent directory. assertEquals(dfsClient.getErasureCodingPolicy(srcFilePath), ecPolicy); - // compare attribute } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java index ad79fd96136..336a7904ef5 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -45,7 +45,6 @@ public void testExecute() throws Exception { unecAction.execute(); assertTrue(unecAction.getExpectedAfterRun()); assertNull(dfsClient.getErasureCodingPolicy(srcPath)); - // compare attribute } @Test From 705759db229652f23636c3a7c42cf8ddf09fdf9c Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 10 Sep 2018 13:44:44 +0800 Subject: [PATCH 09/20] fix wrong check --- .../java/org/smartdata/hdfs/action/ErasureCodingAction.java | 5 ++--- .../org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 1f4903c8ab5..5f1b5a80503 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -85,7 +85,6 @@ protected void execute() throws Exception { this.setDfsClient(HadoopUtil.getDFSClient( HadoopUtil.getNameNodeUri(conf), conf)); // keep attribute consistent - HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); if (fileStatus == null) { throw new ActionException("File doesn't exist!"); @@ -146,11 +145,11 @@ public void validateEcPolicy(String ecPolicyName) throws Exception { for (ErasureCodingPolicyInfo info : dfsClient.getErasureCodingPolicies()) { ecPolicyNameToState.put(info.getPolicy().getName(), info.getState()); } - if (!ecPolicyNameToState.keySet().contains(ecPolicyName) || !ecPolicyName.equals(REPLICATION_POLICY_NAME)) { + if (!ecPolicyNameToState.keySet().contains(ecPolicyName) && !ecPolicyName.equals(REPLICATION_POLICY_NAME)) { throw new ActionException("The EC policy " + ecPolicyName + " is not supported!"); } else if (ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.DISABLED || ecPolicyNameToState.get(ecPolicyName) == ErasureCodingPolicyState.REMOVED) { - throw new ActionException("The EC policy " + ecPolicyName + " is not enabled!"); + throw new ActionException("The EC policy " + ecPolicyName + " is disabled or removed!"); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index eed5c1e98f3..0c5c7d5e4d3 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -132,6 +132,7 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { } catch (MetaStoreException ex) { LOG.error("Error occurred for getting file info", ex); } + // lock the file only if ec or unec action is scheduled fileLock.add(srcPath); return ScheduleResult.SUCCESS; } From b227aafbf9aab8924caaa06724585858efa715d5 Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 10 Sep 2018 14:10:27 +0800 Subject: [PATCH 10/20] fix bugs --- .../hdfs/action/ErasureCodingAction.java | 9 +++++---- .../hdfs/action/ErasureCodingBase.java | 20 ++++++++++++++----- .../hdfs/action/ListErasureCodingPolicy.java | 7 +------ .../hdfs/action/UnErasureCodingAction.java | 8 +++----- .../scheduler/ErasureCodingScheduler.java | 2 ++ 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 5f1b5a80503..86edbf1ac7e 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -32,7 +32,6 @@ import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; -import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -92,7 +91,7 @@ protected void execute() throws Exception { validateEcPolicy(ecPolicyName); ErasureCodingPolicy srcEcPolicy = fileStatus.getErasureCodingPolicy(); // if the current ecPolicy is already the target one, no need to convert - if (srcEcPolicy != null){ + if (srcEcPolicy != null) { if (srcEcPolicy.getName().equals(ecPolicyName)) { appendResult(MATCH_RESULT); this.progress = 1.0F; @@ -115,13 +114,15 @@ protected void execute() throws Exception { HdfsDataOutputStream outputStream = null; try { // a file only with replication policy can be appended. - if (srcEcPolicy.equals(REPLICATION_POLICY_NAME)) { + if (srcEcPolicy == null) { // append the file to acquire the lock to avoid modifying, no real appending occurs. outputStream = dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); } convert(conf, ecPolicyName); - setAttributes(srcPath, ecTmpPath); + // The append operation will change the modification accordingly, + // so we use the filestatus obtained before append to set ecTmp file's most attributes + setAttributes(srcPath, fileStatus, ecTmpPath); dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendResult(CONVERT_RESULT); if (srcEcPolicy == null) { diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index f965f3f3a39..29dbb7813aa 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.smartdata.action.ActionException; import org.smartdata.conf.SmartConf; @@ -33,12 +35,14 @@ import java.util.Map; abstract public class ErasureCodingBase extends HdfsAction { + private static final Logger LOG = + LoggerFactory.getLogger(ErasureCodingBase.class); public static final String BUF_SIZE = "-bufSize"; protected String srcPath; protected String ecTmpPath; protected int bufferSize = 1024 * 1024; protected float progress; - // The values for -ecTmp & -originTmp are assigned by ErasureCodingScheduler. + // The value for -ecTmp is assigned by ErasureCodingScheduler. public static final String EC_TMP = "-ecTmp"; public static final String REPLICATION_POLICY_NAME = SystemErasureCodingPolicies.getReplicationPolicy().getName(); @@ -47,7 +51,8 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti DFSInputStream in = null; DFSOutputStream out = null; try { - long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); + long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); in = dfsClient.open(srcPath, bufferSize, true); HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); // use the same FsPermission as srcPath @@ -85,14 +90,19 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti } // set attributes for dest to keep them consistent with their counterpart of src - protected void setAttributes(String src, String dest) throws IOException { - HdfsFileStatus fileStatus = dfsClient.getFileInfo(src); + protected void setAttributes(String src, HdfsFileStatus fileStatus, String dest) + throws IOException { dfsClient.setOwner(dest, fileStatus.getOwner(), fileStatus.getGroup()); dfsClient.setPermission(dest, fileStatus.getPermission()); dfsClient.setStoragePolicy(dest, dfsClient.getStoragePolicy(src).getName()); // check whether mtime is changed after rename dfsClient.setTimes(dest, fileStatus.getModificationTime(), fileStatus.getAccessTime()); - dfsClient.setAcl(dest, dfsClient.getAclStatus(src).getEntries()); + boolean aclsEnabled = getContext().getConf().getBoolean( + DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, + DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT); + if (aclsEnabled) { + dfsClient.setAcl(dest, dfsClient.getAclStatus(src).getEntries()); + } for(Map.Entry entry : dfsClient.getXAttrs(src).entrySet()) { dfsClient.setXAttr(dest, entry.getKey(), entry.getValue(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java index 8f88d160630..1a75bfe8778 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java @@ -17,14 +17,11 @@ */ package org.smartdata.hdfs.action; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; -import java.util.ArrayList; -import java.util.List; import java.util.Map; @ActionSignature( @@ -45,10 +42,8 @@ public void init(Map args) { public void execute() throws Exception { this.setDfsClient(HadoopUtil.getDFSClient( HadoopUtil.getNameNodeUri(conf), conf)); - List policyInfos = new ArrayList<>(); for (ErasureCodingPolicyInfo policyInfo : dfsClient.getErasureCodingPolicies()) { - policyInfos.add("{" + policyInfo.toString() + "}"); + appendResult("{" + policyInfo.toString() + "}"); } - appendResult(StringUtils.join(policyInfos, ",")); } } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 19f461d3a83..1c0bb6a6422 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -64,8 +64,6 @@ protected void execute() throws Exception { final String CONVERT_RESULT = "The file is converted successfully with replication EC policy."; - // keep attribute consistent - // this.setDfsClient(HadoopUtil.getDFSClient( HadoopUtil.getNameNodeUri(conf), conf)); HdfsFileStatus fileStatus = dfsClient.getFileInfo(srcPath); @@ -87,11 +85,11 @@ protected void execute() throws Exception { } try { convert(conf, ecPolicyName); - setAttributes(srcPath, ecTmpPath); + setAttributes(srcPath, fileStatus, ecTmpPath); dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendResult(CONVERT_RESULT); - appendResult(String.format("The previous EC policy is {}.", srcEcPolicy.getName())); - appendResult(String.format("The current EC policy is {}.", REPLICATION_POLICY_NAME)); + appendResult(String.format("The previous EC policy is %s.", srcEcPolicy.getName())); + appendResult(String.format("The current EC policy is %s.", REPLICATION_POLICY_NAME)); } catch (ActionException ex) { // delete tmp file throw new ActionException(ex); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 0c5c7d5e4d3..9e894ee7362 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -131,6 +131,8 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { } } catch (MetaStoreException ex) { LOG.error("Error occurred for getting file info", ex); + actionInfo.appendLog(ex.getMessage()); + return ScheduleResult.FAIL; } // lock the file only if ec or unec action is scheduled fileLock.add(srcPath); From 5577c14e1585445114ce70adaa33f255876c6d78 Mon Sep 17 00:00:00 2001 From: philo Date: Tue, 11 Sep 2018 11:52:50 +0800 Subject: [PATCH 11/20] fix duplicated record issue after rename --- .../smartdata/hdfs/metric/fetcher/InotifyEventApplier.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index f12df882b01..365e91db4b2 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -234,6 +234,12 @@ private List getRenameSql(Event.RenameEvent renameEvent) if (status == null) { LOG.debug("Get rename dest status failed, {} -> {}", src, dest); } + // The dest path which the src is renamed to should be checked in file table + // to avoid duplicated record for one same path. + FileInfo destInfo = metaStore.getFile(dest); + if (destInfo != null) { + metaStore.deleteFileByPath(dest); + } if (info == null) { if (status != null) { info = HadoopUtil.convertFileStatus(status, dest); From 8c7c6ab5ee6fa76584a5296c578af815149892af Mon Sep 17 00:00:00 2001 From: philo Date: Tue, 11 Sep 2018 14:23:28 +0800 Subject: [PATCH 12/20] remove StripErasureCodeFileAction.java --- .../action/StripErasureCodeFileAction.java | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/StripErasureCodeFileAction.java diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/StripErasureCodeFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/StripErasureCodeFileAction.java deleted file mode 100644 index 2be287215ee..00000000000 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/StripErasureCodeFileAction.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.action; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * An action to do strip level erasure code a file. Only for Hadoop 3.x. - */ -public class StripErasureCodeFileAction extends HdfsAction { - private static final Logger LOG = LoggerFactory.getLogger(StripErasureCodeFileAction.class); - - @Override - protected void execute() throws Exception { - } -} From 5fb718bbe6f4ddd8f09b5b70a80e4ac6b26f654b Mon Sep 17 00:00:00 2001 From: philo Date: Tue, 11 Sep 2018 15:22:34 +0800 Subject: [PATCH 13/20] fix compatibility issue for ec related actions --- .../hdfs/action/HdfsActionFactory.java | 27 ++++++++++++++++--- .../scheduler/ErasureCodingScheduler.java | 19 +++++-------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java index 690b9ef24a8..55a0ec5caa2 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/HdfsActionFactory.java @@ -17,13 +17,25 @@ */ package org.smartdata.hdfs.action; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.smartdata.action.AbstractActionFactory; +import org.smartdata.action.SmartAction; import org.smartdata.hdfs.scheduler.ErasureCodingScheduler; +import java.util.Arrays; +import java.util.List; + /** * Built-in smart actions for HDFS system. */ public class HdfsActionFactory extends AbstractActionFactory { + private static final Logger LOG = LoggerFactory.getLogger(HdfsActionFactory.class); + public static final List HDFS3_ACTION_CLASSES = Arrays.asList( + "org.smartdata.hdfs.action.ListErasureCodingPolicy", + "org.smartdata.hdfs.action.CheckErasureCodingPolicy", + "org.smartdata.hdfs.action.ErasureCodingAction", + "org.smartdata.hdfs.action.UnErasureCodingAction"); static { addAction(AllSsdFileAction.class); @@ -58,10 +70,17 @@ public class HdfsActionFactory extends AbstractActionFactory { // addAction("clusterbalance", ClusterBalanceAction.class); // addAction("setstoragepolicy", SetStoragePolicyAction.class); if (ErasureCodingScheduler.isECSupported()) { - addAction(ListErasureCodingPolicy.class); - addAction(CheckErasureCodingPolicy.class); - addAction(ErasureCodingAction.class); - addAction(UnErasureCodingAction.class); + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + if (loader == null) { + loader = ClassLoader.getSystemClassLoader(); + } + try { + for (String classString : HDFS3_ACTION_CLASSES) { + addAction((Class) loader.loadClass(classString)); + } + } catch (ClassNotFoundException ex) { + LOG.error("Class not found!", ex); + } } } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 9e894ee7362..40a01249bd5 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -38,11 +38,12 @@ public class ErasureCodingScheduler extends ActionSchedulerService { public static final Logger LOG = LoggerFactory.getLogger(ErasureCodingScheduler.class); - private String ecActionID; - private String unecActionID; - private String checkecActionID; - private String listecActionID; - private List actions; + public static final String ecActionID = "ec"; + public static final String unecActionID = "unec"; + public static final String checkecActionID = "checkec"; + public static final String listecActionID = "listec"; + public static final List actions = + Arrays.asList(ecActionID, unecActionID, checkecActionID, listecActionID); public static final String EC_DIR = "/system/ssm/ec_tmp/"; public static final String EC_TMP = "-ecTmp"; private Set fileLock; @@ -53,14 +54,6 @@ public ErasureCodingScheduler(SmartContext context, MetaStore metaStore) { super(context, metaStore); this.conf = context.getConf(); this.metaStore = metaStore; - - if (isECSupported()) { - ecActionID = ErasureCodingAction.class.getAnnotation(ActionSignature.class).actionId(); - unecActionID = UnErasureCodingAction.class.getAnnotation(ActionSignature.class).displayName(); - checkecActionID = CheckErasureCodingPolicy.class.getAnnotation(ActionSignature.class).displayName(); - listecActionID = ListErasureCodingPolicy.class.getAnnotation(ActionSignature.class).displayName(); - actions = Arrays.asList(ecActionID, unecActionID, checkecActionID, listecActionID); - } } public List getSupportedActions() { return actions; From 2b6dbd21028990be5eac7c8fa078d14b8d608f9c Mon Sep 17 00:00:00 2001 From: philo Date: Wed, 12 Sep 2018 15:27:34 +0800 Subject: [PATCH 14/20] record EC policy ID in metastore --- .../java/org/smartdata/model/FileInfo.java | 20 ++++++++++++++-- .../org/smartdata/model/TestFileInfo.java | 24 ++++++++++++------- .../smartdata/hdfs/CompatibilityHelper2.java | 5 ++++ .../smartdata/hdfs/CompatibilityHelper31.java | 10 ++++++++ .../hdfs/action/ErasureCodingBase.java | 1 + .../smartdata/hdfs/CompatibilityHelper.java | 2 ++ .../java/org/smartdata/hdfs/HadoopUtil.java | 1 + .../smartdata/hdfs/action/MetaDataAction.java | 3 ++- .../metric/fetcher/InotifyEventApplier.java | 8 ++++++- .../hdfs/metric/fetcher/NamespaceFetcher.java | 4 +++- .../metric/fetcher/TestCachedListFetcher.java | 3 ++- .../org/smartdata/metastore/MetaStore.java | 4 ++-- .../smartdata/metastore/dao/FileInfoDao.java | 4 +++- .../smartdata/metastore/TestMetaStore.java | 12 +++++++--- .../dao/TestAccessCountTableManager.java | 2 ++ .../metastore/dao/TestFileInfoDao.java | 14 ++++++----- .../metastore/dao/TestTableAggregator.java | 1 + .../server/engine/rule/TestRuleManager.java | 2 +- 18 files changed, 93 insertions(+), 27 deletions(-) diff --git a/smart-common/src/main/java/org/smartdata/model/FileInfo.java b/smart-common/src/main/java/org/smartdata/model/FileInfo.java index 3e119cb600f..88b5be5622c 100644 --- a/smart-common/src/main/java/org/smartdata/model/FileInfo.java +++ b/smart-common/src/main/java/org/smartdata/model/FileInfo.java @@ -32,11 +32,12 @@ public class FileInfo { private String owner; private String group; private byte storagePolicy; + private byte erasureCodingPolicy; public FileInfo(String path, long fileId, long length, boolean isdir, short blockReplication, long blocksize, long modificationTime, long accessTime, short permission, String owner, String group, - byte storagePolicy) { + byte storagePolicy, byte erasureCodingPolicy) { this.path = path; this.fileId = fileId; this.length = length; @@ -49,6 +50,7 @@ public FileInfo(String path, long fileId, long length, boolean isdir, this.owner = owner; this.group = group; this.storagePolicy = storagePolicy; + this.erasureCodingPolicy = erasureCodingPolicy; } public String getPath() { @@ -147,6 +149,14 @@ public void setStoragePolicy(byte storagePolicy) { this.storagePolicy = storagePolicy; } + public byte getErasureCodingPolicy() { + return erasureCodingPolicy; + } + + public void setErasureCodingPolicy(byte erasureCodingPolicy) { + this.erasureCodingPolicy = erasureCodingPolicy; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -224,6 +234,7 @@ public static class Builder { private String owner; private String group; private byte storagePolicy; + private byte erasureCodingPolicy; public Builder setPath(String path) { this.path = path; @@ -285,10 +296,15 @@ public Builder setStoragePolicy(byte storagePolicy) { return this; } + public Builder setErasureCodingPolicy(byte erasureCodingPolicy) { + this.erasureCodingPolicy = erasureCodingPolicy; + return this; + } + public FileInfo build() { return new FileInfo(path, fileId, length, isdir, blockReplication, blocksize, modificationTime, accessTime, permission, owner, - group, storagePolicy); + group, storagePolicy, erasureCodingPolicy); } @Override diff --git a/smart-common/src/test/java/org/smartdata/model/TestFileInfo.java b/smart-common/src/test/java/org/smartdata/model/TestFileInfo.java index cacaf219909..269b119603a 100644 --- a/smart-common/src/test/java/org/smartdata/model/TestFileInfo.java +++ b/smart-common/src/test/java/org/smartdata/model/TestFileInfo.java @@ -36,7 +36,8 @@ public ChildFileInfo( short permission, String owner, String group, - byte storagePolicy) { + byte storagePolicy, + byte erasureCodingPolicy) { super( path, fileId, @@ -49,7 +50,8 @@ public ChildFileInfo( permission, owner, group, - storagePolicy); + storagePolicy, + erasureCodingPolicy); } } @@ -57,37 +59,43 @@ public ChildFileInfo( public void testEquals() throws Exception { //Case 1: FileInfo fileInfo = - new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, 1, (short) 1, " ", " ", (byte) 1); + new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, + 1, (short) 1, " ", " ", (byte) 1, (byte) 0); Assert.assertEquals(true, fileInfo.equals(fileInfo)); //Case 2: FileInfo fileInfo1 = - new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, 1, (short) 1, " ", " ", (byte) 1); + new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, + 1, (short) 1, " ", " ", (byte) 1, (byte) 0); Assert.assertEquals(true, fileInfo.equals(fileInfo1)); //Case 3: FileInfo fileInfo2 = - new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, 1, (short) 1, null, " ", (byte) 1); + new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, + 1, (short) 1, null, " ", (byte) 1, (byte) 0); Assert.assertEquals(false, fileInfo.equals(fileInfo2)); Assert.assertEquals(false, fileInfo2.equals(fileInfo)); //Case 4: FileInfo fileInfo3 = - new FileInfo(null, 1, 1, true, (short) 1, 1, 1, 1, (short) 1, " ", " ", (byte) 1); + new FileInfo(null, 1, 1, true, (short) 1, 1, 1, + 1, (short) 1, " ", " ", (byte) 1, (byte) 0); Assert.assertEquals(false, fileInfo.equals(fileInfo3)); Assert.assertEquals(false, fileInfo3.equals(fileInfo)); //Case 5: FileInfo fileInfo4 = - new FileInfo(null, 1, 1, true, (short) 1, 1, 1, 1, (short) 1, " ", null, (byte) 1); + new FileInfo(null, 1, 1, true, (short) 1, 1, 1, + 1, (short) 1, " ", null, (byte) 1, (byte) 0); Assert.assertEquals(false, fileInfo.equals(fileInfo4)); Assert.assertEquals(false, fileInfo4.equals(fileInfo)); //Case 6: FileInfo fileInfo5 = - new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, 1, (short) 2, " ", " ", (byte) 1); + new FileInfo(" ", 1, 1, true, (short) 1, 1, 1, + 1, (short) 2, " ", " ", (byte) 1, (byte) 0); Assert.assertEquals(false, fileInfo.equals(fileInfo5)); } } diff --git a/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java index 935321b452e..daafdf2eb7a 100644 --- a/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java +++ b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java @@ -62,4 +62,9 @@ public HdfsFileStatus createHdfsFileStatus( length, isdir, block_replication, blocksize, modification_time, access_time, permission, owner, group, symlink, path, fileId, childrenNum, feInfo, storagePolicy); } + + public byte getErasureCodingPolicy(HdfsFileStatus fileStatus) { + // for HDFS2.x, the erasure policy is always replication whose id is 0 in HDFS. + return (byte) 0; + } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java index a1ce43f0205..646487d202d 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java @@ -207,4 +207,14 @@ public HdfsFileStatus createHdfsFileStatus( .storagePolicy(storagePolicy) .build(); } + + @Override + public byte getErasureCodingPolicy(HdfsFileStatus fileStatus) { + ErasureCodingPolicy erasureCodingPolicy = fileStatus.getErasureCodingPolicy(); + // null means replication policy and its id is 0 in HDFS. + if (erasureCodingPolicy == null) { + return (byte) 0; + } + return fileStatus.getErasureCodingPolicy().getId(); + } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 29dbb7813aa..fea98bdcb07 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -103,6 +103,7 @@ protected void setAttributes(String src, HdfsFileStatus fileStatus, String dest) if (aclsEnabled) { dfsClient.setAcl(dest, dfsClient.getAclStatus(src).getEntries()); } + //TODO: check ec related record to avoid paradox for(Map.Entry entry : dfsClient.getXAttrs(src).entrySet()) { dfsClient.setXAttr(dest, entry.getKey(), entry.getValue(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); diff --git a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java index aad542a1b20..548599c562a 100644 --- a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java @@ -88,4 +88,6 @@ HdfsFileStatus createHdfsFileStatus( long length, boolean isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy); + + byte getErasureCodingPolicy(HdfsFileStatus fileStatus); } diff --git a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java index 6ccd040c925..febf610f025 100644 --- a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/HadoopUtil.java @@ -254,6 +254,7 @@ public static FileInfo convertFileStatus(HdfsFileStatus status, String path) { .setOwner(status.getOwner()) .setGroup(status.getGroup()) .setStoragePolicy(status.getStoragePolicy()) + .setErasureCodingPolicy(CompatibilityHelperLoader.getHelper().getErasureCodingPolicy(status)) .build(); } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java index a816b99dfa9..169acf66614 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/MetaDataAction.java @@ -111,7 +111,8 @@ protected void execute() throws Exception { permission, ownerName, groupName, - (byte) 1); + (byte) 1, + (byte) 0); changeFileMetaData(srcPath, fileInfo); } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index 365e91db4b2..2cd8b7d3dc4 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -59,7 +59,7 @@ public void apply(List events) throws IOException, MetaStoreException { List statements = new ArrayList<>(); for (Event event : events) { List gen = getSqlStatement(event); - if (gen != null && !gen.isEmpty()){ + if (gen != null && !gen.isEmpty()) { for (String s : gen) { if (s != null && s.length() > 0) { statements.add(s); @@ -341,6 +341,7 @@ private String getMetaDataUpdateSql(Event.MetadataUpdateEvent metadataUpdateEven "UPDATE file SET block_replication = %s WHERE path = '%s';", metadataUpdateEvent.getReplication(), metadataUpdateEvent.getPath()); case XATTRS: + final String EC_POLICY = "hdfs.erasurecoding.policy"; //Todo if (LOG.isDebugEnabled()) { String message = "\n"; @@ -349,6 +350,11 @@ private String getMetaDataUpdateSql(Event.MetadataUpdateEvent metadataUpdateEven } LOG.debug(message); } + for (XAttr xAttr : metadataUpdateEvent.getxAttrs()) { + if (xAttr.getName().equals(EC_POLICY)) { + return String.format("UPDATE "); + } + } break; case ACLS: return ""; diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java index 7d194101f1c..79057040eb4 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/NamespaceFetcher.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import org.smartdata.conf.SmartConf; import org.smartdata.conf.SmartConfKeys; +import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.metastore.MetaStoreException; import org.smartdata.model.FileInfo; import org.smartdata.metastore.MetaStore; @@ -307,7 +308,8 @@ private FileInfo convertToFileInfo(HdfsFileStatus status, String parent) { status.getPermission().toShort(), status.getOwner(), status.getGroup(), - status.getStoragePolicy()); + status.getStoragePolicy(), + CompatibilityHelperLoader.getHelper().getErasureCodingPolicy(status)); return fileInfo; } } diff --git a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java index b2285f36126..198c40c20ed 100644 --- a/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java +++ b/smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/metric/fetcher/TestCachedListFetcher.java @@ -105,10 +105,11 @@ private FileInfo createFileStatus(String pathString) { String group = "admin"; long fileId = fid; byte storagePolicy = 0; + byte erasureCodingPolicy = 0; fid++; return new FileInfo(pathString, fileId, length, isDir, (short)blockReplication, blockSize, modTime, accessTime, - (short) 1, owner, group, storagePolicy); + (short) 1, owner, group, storagePolicy, erasureCodingPolicy); } @Test diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java index 18a86717664..958bffaeb31 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java @@ -686,7 +686,7 @@ public List listFileActions(long rid, // Add a mock fileInfo fileInfo = new FileInfo(filePath, 0L, 0L, false, (short) 0, 0L, 0L, 0L, (short) 0, - "root", "root", (byte) 0); + "root", "root", (byte) 0, (byte) 0); } detailedFileAction.setFileLength(fileInfo.getLength()); detailedFileAction.setFilePath(filePath); @@ -723,7 +723,7 @@ public List listFileActions(long rid, long start, long offse // Add a mock fileInfo fileInfo = new FileInfo(filePath, 0L, 0L, false, (short) 0, 0L, 0L, 0L, (short) 0, - "root", "root", (byte) 0); + "root", "root", (byte) 0, (byte) 0); } detailedFileAction.setFileLength(fileInfo.getLength()); detailedFileAction.setFilePath(filePath); diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java index 852d1c4ddb9..276b89070b7 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/dao/FileInfoDao.java @@ -173,6 +173,7 @@ private Map toMap(FileInfo fileInfo) { parameters .put("owner_group", fileInfo.getGroup()); parameters.put("permission", fileInfo.getPermission()); + parameters.put("ec_policy_id", fileInfo.getErasureCodingPolicy()); return parameters; } @@ -191,7 +192,8 @@ public FileInfo mapRow(ResultSet resultSet, int i) resultSet.getShort("permission"), resultSet.getString("owner"), resultSet.getString("owner_group"), - resultSet.getByte("sid") + resultSet.getByte("sid"), + resultSet.getByte("ec_policy_id") ); return fileInfo; } diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/TestMetaStore.java b/smart-metastore/src/test/java/org/smartdata/metastore/TestMetaStore.java index a4d43195199..fa3bf8d44f0 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/TestMetaStore.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/TestMetaStore.java @@ -173,6 +173,7 @@ public void testGetFiles() throws Exception { String group = "admin"; long fileId = 56L; byte storagePolicy = 0; + byte erasureCodingPolicy = 0; FileInfo fileInfo = new FileInfo( pathString, @@ -186,7 +187,8 @@ public void testGetFiles() throws Exception { (short) 1, owner, group, - storagePolicy); + storagePolicy, + erasureCodingPolicy); metaStore.insertFile(fileInfo); FileInfo dbFileInfo = metaStore.getFile(56); Assert.assertTrue(dbFileInfo.equals(fileInfo)); @@ -292,6 +294,7 @@ public void testMoveSyncRules() throws Exception { String group = "admin"; long fileId = 56L; byte storagePolicy = 0; + byte erasureCodingPolicy = 0; FileInfo fileInfo = new FileInfo( pathString, @@ -305,7 +308,8 @@ public void testMoveSyncRules() throws Exception { (short) 1, owner, group, - storagePolicy); + storagePolicy, + erasureCodingPolicy); metaStore.insertFile(fileInfo); Map args = new HashMap(); args.put("-file", "/src/1"); @@ -417,6 +421,7 @@ public void testInsetFiles() throws Exception { String group = "admin"; long fileId = 312321L; byte storagePolicy = 0; + byte erasureCodingPolicy = 0; FileInfo[] files = { new FileInfo( pathString, @@ -430,7 +435,8 @@ public void testInsetFiles() throws Exception { (short) 1, owner, group, - storagePolicy) + storagePolicy, + erasureCodingPolicy) }; metaStore.insertFiles(files); FileInfo dbFileInfo = metaStore.getFile("/tmp/testFile"); diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestAccessCountTableManager.java b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestAccessCountTableManager.java index ce643d69269..491ac35a747 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestAccessCountTableManager.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestAccessCountTableManager.java @@ -172,6 +172,7 @@ private void prepareFiles(MetaStore metaStore) throws MetaStoreException { (short) 1, "root", "admin", + (byte) 0, (byte) 0)); } metaStore.insertFiles(statusInternals.toArray(new FileInfo[0])); @@ -190,6 +191,7 @@ private void insertNewFile(MetaStore metaStore, String file, Long fid) (short) 1, "root", "admin", + (byte) 0, (byte) 0); metaStore.insertFile(finfo); } diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileInfoDao.java b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileInfoDao.java index 694b5d73c56..2980939f7f4 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileInfoDao.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestFileInfoDao.java @@ -57,15 +57,16 @@ public void testInsetGetDeleteFiles() throws Exception { String group = "admin"; long fileId = 312321L; byte storagePolicy = 0; - FileInfo fileInfo = new FileInfo(path, fileId, length, isDir, blockReplication, - blockSize, modTime, accessTime, permission, owner, group, storagePolicy); + byte erasureCodingPolicy = 0; + FileInfo fileInfo = new FileInfo(path, fileId, length, isDir, blockReplication, blockSize, + modTime, accessTime, permission, owner, group, storagePolicy, erasureCodingPolicy); fileInfoDao.insert(fileInfo); FileInfo file1 = fileInfoDao.getByPath("/testFile"); Assert.assertTrue(fileInfo.equals(file1)); FileInfo file2 = fileInfoDao.getById(fileId); Assert.assertTrue(fileInfo.equals(file2)); - FileInfo fileInfo1 = new FileInfo(path, fileId + 1, length, isDir, blockReplication, - blockSize, modTime, accessTime, permission, owner, group, storagePolicy); + FileInfo fileInfo1 = new FileInfo(path, fileId + 1, length, isDir, blockReplication, blockSize, + modTime, accessTime, permission, owner, group, storagePolicy, erasureCodingPolicy); fileInfoDao.insert(fileInfo1); List fileInfos = fileInfoDao.getFilesByPrefix("/testaaFile"); Assert.assertTrue(fileInfos.size() == 0); @@ -93,12 +94,13 @@ public void testInsertUpdateFiles() throws Exception { String group = "admin"; long fileId = 312321L; byte storagePolicy = 0; + byte erasureCodingPolicy = 0; Map mapOwnerIdName = new HashMap<>(); mapOwnerIdName.put(1, "root"); Map mapGroupIdName = new HashMap<>(); mapGroupIdName.put(1, "admin"); - FileInfo fileInfo = new FileInfo(path, fileId, length, isDir, blockReplication, - blockSize, modTime, accessTime, permission, owner, group, storagePolicy); + FileInfo fileInfo = new FileInfo(path, fileId, length, isDir, blockReplication, blockSize, + modTime, accessTime, permission, owner, group, storagePolicy, erasureCodingPolicy); fileInfoDao.insert(fileInfo); fileInfoDao.update(path, 10); FileInfo file = fileInfoDao.getById(fileId); diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestTableAggregator.java b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestTableAggregator.java index 9ea42113767..6e31aad6b7f 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestTableAggregator.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/dao/TestTableAggregator.java @@ -109,6 +109,7 @@ private void prepareFiles(MetaStore metaStore) throws MetaStoreException { (short) 1, "root", "admin", + (byte) 0, (byte) 0)); } metaStore.insertFiles(statusInternals.toArray(new FileInfo[0])); diff --git a/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java b/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java index 1b8c14691ed..7df4ed80423 100644 --- a/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java +++ b/smart-server/src/test/java/org/smartdata/server/engine/rule/TestRuleManager.java @@ -303,7 +303,7 @@ public void testMultiThreadChangeState() throws Exception { long length = 100; long fid = 10000; FileInfo[] files = { new FileInfo("/tmp/testfile", fid, length, false, (short) 3, - 1024, now, now, (short) 1, null, null, (byte) 3) }; + 1024, now, now, (short) 1, null, null, (byte) 3, (byte) 0) }; metaStore.insertFiles(files); long rid = ruleManager.submitRule(rule, RuleState.ACTIVE); From 8e56068d4960adbc15a61f5d8e14e8afb3f23cc3 Mon Sep 17 00:00:00 2001 From: philo Date: Wed, 12 Sep 2018 15:28:37 +0800 Subject: [PATCH 15/20] update EC policy for directory --- .../smartdata/hdfs/CompatibilityHelper2.java | 5 +++++ .../smartdata/hdfs/CompatibilityHelper31.java | 13 +++++++++++++ .../smartdata/hdfs/CompatibilityHelper.java | 2 ++ .../metric/fetcher/InotifyEventApplier.java | 19 ++++++++++++++++++- 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java index daafdf2eb7a..9c8d31fe66a 100644 --- a/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java +++ b/smart-hadoop-support/smart-hadoop-2/src/main/java/org/smartdata/hdfs/CompatibilityHelper2.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.protocol.*; import org.apache.hadoop.hdfs.protocolPB.PBHelper; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; @@ -67,4 +68,8 @@ public byte getErasureCodingPolicy(HdfsFileStatus fileStatus) { // for HDFS2.x, the erasure policy is always replication whose id is 0 in HDFS. return (byte) 0; } + + public byte getErasureCodingPolicyByName(DFSClient client, String ecPolicyName) throws IOException { + return (byte) 0; + } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java index 646487d202d..c78723a864f 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/CompatibilityHelper31.java @@ -217,4 +217,17 @@ public byte getErasureCodingPolicy(HdfsFileStatus fileStatus) { } return fileStatus.getErasureCodingPolicy().getId(); } + + @Override + public byte getErasureCodingPolicyByName(DFSClient client, String ecPolicyName) throws IOException { + if (ecPolicyName.equals(SystemErasureCodingPolicies.getReplicationPolicy().getName())) { + return (byte) 0; + } + for (ErasureCodingPolicyInfo policyInfo : client.getErasureCodingPolicies()) { + if (policyInfo.getPolicy().getName().equals(ecPolicyName)) { + return policyInfo.getPolicy().getId(); + } + } + return (byte) -1; + } } \ No newline at end of file diff --git a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java index 548599c562a..6cb4e0d1b9a 100644 --- a/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java +++ b/smart-hadoop-support/smart-hadoop-common/src/main/java/org/smartdata/hdfs/CompatibilityHelper.java @@ -90,4 +90,6 @@ HdfsFileStatus createHdfsFileStatus( long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy); byte getErasureCodingPolicy(HdfsFileStatus fileStatus); + + byte getErasureCodingPolicyByName(DFSClient client, String ecPolicyName) throws IOException; } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java index 2cd8b7d3dc4..c419d7ec981 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/InotifyEventApplier.java @@ -21,8 +21,10 @@ import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.inotify.Event; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.io.WritableUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.hdfs.HadoopUtil; import org.smartdata.metastore.DBType; import org.smartdata.metastore.MetaStore; @@ -32,6 +34,8 @@ import org.smartdata.model.FileDiffType; import org.smartdata.model.FileInfo; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -350,9 +354,22 @@ private String getMetaDataUpdateSql(Event.MetadataUpdateEvent metadataUpdateEven } LOG.debug(message); } + // The following code should be executed merely on HDFS3.x. for (XAttr xAttr : metadataUpdateEvent.getxAttrs()) { if (xAttr.getName().equals(EC_POLICY)) { - return String.format("UPDATE "); + try { + String ecPolicyName = WritableUtils.readString( + new DataInputStream(new ByteArrayInputStream(xAttr.getValue()))); + byte ecPolicyId = CompatibilityHelperLoader.getHelper(). + getErasureCodingPolicyByName(client, ecPolicyName); + if (ecPolicyId == (byte) -1) { + LOG.error("Unrecognized EC policy for updating!"); + } + return String.format("UPDATE file SET ec_policy_id = %s WHERE path = '%s'", + ecPolicyId, metadataUpdateEvent.getPath()); + } catch (IOException ex) { + LOG.error("Error occurred for updating ecPolicy!", ex); + } } } break; From e74c1ac0ae6ebdd0871384e462fc8a1fd0d7f521 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 13 Sep 2018 16:35:09 +0800 Subject: [PATCH 16/20] complement EC info in FileInfo --- .../main/java/org/smartdata/model/FileInfo.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/smart-common/src/main/java/org/smartdata/model/FileInfo.java b/smart-common/src/main/java/org/smartdata/model/FileInfo.java index 88b5be5622c..1cab069c27f 100644 --- a/smart-common/src/main/java/org/smartdata/model/FileInfo.java +++ b/smart-common/src/main/java/org/smartdata/model/FileInfo.java @@ -175,6 +175,7 @@ public boolean equals(Object o) { && accessTime == fileInfo.accessTime && permission == fileInfo.permission && storagePolicy == fileInfo.storagePolicy + && erasureCodingPolicy == fileInfo.erasureCodingPolicy && Objects.equals(path, fileInfo.path) && Objects.equals(owner, fileInfo.owner) && Objects.equals(group, fileInfo.group); @@ -194,7 +195,8 @@ public int hashCode() { permission, owner, group, - storagePolicy); + storagePolicy, + erasureCodingPolicy); } public static Builder newBuilder() { @@ -206,7 +208,7 @@ public String toString() { return String.format( "FileInfo{path=\'%s\', fileId=%s, length=%s, isdir=%s, blockReplication=%s, " + "blocksize=%s, modificationTime=%s, accessTime=%s, permission=%s, owner=\'%s\', " - + "group=\'%s\', storagePolicy=%s}", + + "group=\'%s\', storagePolicy=%s, erasureCodingPolicy=%s}", path, fileId, length, @@ -218,7 +220,8 @@ public String toString() { permission, owner, group, - storagePolicy); + storagePolicy, + erasureCodingPolicy); } public static class Builder { @@ -312,7 +315,7 @@ public String toString() { return String.format( "Builder{path=\'%s\', fileId=%s, length=%s, isdir=%s, blockReplication=%s, " + "blocksize=%s, modificationTime=%s, accessTime=%s, permission=%s, owner=\'%s\', " - + "group=\'%s\', storagePolicy=\'%s\'}", + + "group=\'%s\', storagePolicy=%s, erasureCodingPolicy=%s}", path, fileId, length, @@ -324,7 +327,8 @@ public String toString() { permission, owner, group, - storagePolicy); + storagePolicy, + erasureCodingPolicy); } } } From 259e97d2376a7ddb3bec0a2a68d5979066a86c22 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 13 Sep 2018 17:25:30 +0800 Subject: [PATCH 17/20] refine the code --- .../hdfs/action/CheckErasureCodingPolicy.java | 3 +++ .../hdfs/action/ErasureCodingAction.java | 23 +++++++++++++++---- .../hdfs/action/ErasureCodingBase.java | 6 +++-- .../hdfs/action/ListErasureCodingPolicy.java | 3 +++ .../hdfs/action/UnErasureCodingAction.java | 15 ++++++++++-- .../scheduler/ErasureCodingScheduler.java | 7 +++--- 6 files changed, 45 insertions(+), 12 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java index cfabd5bb1c4..84074bc3e73 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/CheckErasureCodingPolicy.java @@ -24,6 +24,9 @@ import java.util.Map; +/** + * An action to check the EC policy for a file or dir. + */ @ActionSignature( actionId = "checkec", displayName = "checkec", diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 86edbf1ac7e..7f732835851 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -32,10 +32,15 @@ import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; +import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; +/** + * An action to set an EC policy for a dir or convert a file to another one in an EC policy. + * Default values are used for arguments of policy & bufSize if their values are not given in this action. + */ @ActionSignature( actionId = "ec", displayName = "ec", @@ -59,14 +64,14 @@ public void init(Map args) { // this is a temp file kept for converting a file to another with other ec policy. this.ecTmpPath = args.get(EC_TMP); } - if (args.containsKey(EC_POLICY_NAME)) { + if (args.containsKey(EC_POLICY_NAME) && !args.get(EC_POLICY_NAME).isEmpty()) { this.ecPolicyName = args.get(EC_POLICY_NAME); } else { String defaultEcPolicy = conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY, DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT); this.ecPolicyName = defaultEcPolicy; } - if (args.containsKey(BUF_SIZE)) { + if (args.containsKey(BUF_SIZE) && !args.get(BUF_SIZE).isEmpty()) { this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); } this.progress = 0.0F; @@ -120,8 +125,10 @@ protected void execute() throws Exception { dfsClient.append(srcPath, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); } convert(conf, ecPolicyName); - // The append operation will change the modification accordingly, - // so we use the filestatus obtained before append to set ecTmp file's most attributes + /** + * The append operation will change the modification accordingly, + * so we use the filestatus obtained before append to set ecTmp file's most attributes + */ setAttributes(srcPath, fileStatus, ecTmpPath); dfsClient.rename(ecTmpPath, srcPath, Options.Rename.OVERWRITE); appendResult(CONVERT_RESULT); @@ -132,7 +139,13 @@ protected void execute() throws Exception { } appendResult(String.format("The current EC policy is {}.", ecPolicyName)); } catch (ActionException ex) { - // delete tmp file + try { + if (dfsClient.getFileInfo(ecTmpPath) != null) { + dfsClient.delete(ecTmpPath, false); + } + } catch (IOException e) { + LOG.error("Failed to delete tmp file created during the conversion!"); + } throw new ActionException(ex); } finally { if (outputStream != null) { diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index fea98bdcb07..91ecdce9a1b 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -34,6 +34,9 @@ import java.util.EnumSet; import java.util.Map; +/** + * An abstract base class for ErasureCodingAction & UnErasureCodingAction. + */ abstract public class ErasureCodingBase extends HdfsAction { private static final Logger LOG = LoggerFactory.getLogger(ErasureCodingBase.class); @@ -84,7 +87,7 @@ protected void convert(SmartConf conf, String ecPolicyName) throws ActionExcepti out.close(); } } catch (IOException ex) { - throw new ActionException(ex); + LOG.error("IOException occurred when closing DFSInputStream or DFSOutputStream!"); } } } @@ -95,7 +98,6 @@ protected void setAttributes(String src, HdfsFileStatus fileStatus, String dest) dfsClient.setOwner(dest, fileStatus.getOwner(), fileStatus.getGroup()); dfsClient.setPermission(dest, fileStatus.getPermission()); dfsClient.setStoragePolicy(dest, dfsClient.getStoragePolicy(src).getName()); - // check whether mtime is changed after rename dfsClient.setTimes(dest, fileStatus.getModificationTime(), fileStatus.getAccessTime()); boolean aclsEnabled = getContext().getConf().getBoolean( DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java index 1a75bfe8778..7ad70b77cf8 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ListErasureCodingPolicy.java @@ -24,6 +24,9 @@ import java.util.Map; +/** + * An action to list the info for all EC policies in HDFS. + */ @ActionSignature( actionId = "listec", displayName = "listec", diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java index 1c0bb6a6422..8d05c2f1f44 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/UnErasureCodingAction.java @@ -27,8 +27,13 @@ import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.HadoopUtil; +import java.io.IOException; import java.util.Map; +/** + * An action to set replication policy for a dir or convert a file to another one in replication policy. + * Default value is used for argument of bufSize if its value is not given in this action. + */ @ActionSignature( actionId = "unec", displayName = "unec", @@ -49,7 +54,7 @@ public void init(Map args) { if (args.containsKey(EC_TMP)) { this.ecTmpPath = args.get(EC_TMP); } - if (args.containsKey(BUF_SIZE)) { + if (args.containsKey(BUF_SIZE) && !args.get(BUF_SIZE).isEmpty()) { this.bufferSize = Integer.valueOf(args.get(BUF_SIZE)); } this.progress = 0.0F; @@ -91,7 +96,13 @@ protected void execute() throws Exception { appendResult(String.format("The previous EC policy is %s.", srcEcPolicy.getName())); appendResult(String.format("The current EC policy is %s.", REPLICATION_POLICY_NAME)); } catch (ActionException ex) { - // delete tmp file + try { + if (dfsClient.getFileInfo(ecTmpPath) != null) { + dfsClient.delete(ecTmpPath, false); + } + } catch (IOException e) { + LOG.error("Failed to delete tmp file created during the conversion!"); + } throw new ActionException(ex); } } diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 40a01249bd5..bf17657efa4 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -21,7 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.SmartContext; -import org.smartdata.action.annotation.ActionSignature; import org.smartdata.conf.SmartConf; import org.smartdata.hdfs.action.*; import org.smartdata.metastore.MetaStore; @@ -143,8 +142,10 @@ private String createTmpName(LaunchAction action) { } else { fileName = path.substring(index + 1, path.length()); } - // The dest tmp file is under EC_DIR and - // named by fileName, aidxxx and current time in millisecond with "_" separated + /** + * The dest tmp file is under EC_DIR and + * named by fileName, aidxxx and current time in millisecond with "_" separated + */ String tmpName = fileName + "_" + "aid" + action.getActionId() + "_" + System.currentTimeMillis(); return tmpName; From 4b720bcd19522bdf49f7fe7eb528fb2e2456816e Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 14 Sep 2018 10:42:25 +0800 Subject: [PATCH 18/20] modify unit test --- smart-hadoop-support/smart-hadoop-3.1/pom.xml | 17 +++++ .../hdfs/action/ErasureCodingAction.java | 6 +- .../hdfs/action/TestErasureCodingAction.java | 8 +- .../action/TestErasureCodingActionBase.java | 45 ----------- .../action/TestErasureCodingMiniCluster.java | 75 +++++++++++++++++++ .../action/TestUnErasureCodingAction.java | 22 +++--- 6 files changed, 114 insertions(+), 59 deletions(-) delete mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java create mode 100644 smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingMiniCluster.java diff --git a/smart-hadoop-support/smart-hadoop-3.1/pom.xml b/smart-hadoop-support/smart-hadoop-3.1/pom.xml index 2a37aa8c4cd..b371795c186 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/pom.xml +++ b/smart-hadoop-support/smart-hadoop-3.1/pom.xml @@ -89,6 +89,23 @@ test test-jar + + org.mockito + mockito-all + test + + + org.apache.hadoop + hadoop-common + test + test-jar + + + org.apache.hadoop + hadoop-hdfs + test + test-jar + junit junit diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java index 7f732835851..2b50d9a36ce 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingAction.java @@ -149,7 +149,11 @@ protected void execute() throws Exception { throw new ActionException(ex); } finally { if (outputStream != null) { - outputStream.close(); + try { + outputStream.close(); + } catch (IOException ex) { + // IOException may be reported for missing blocks. + } } } } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java index 2189abfcfe3..baeb560d20a 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -24,7 +24,7 @@ import java.util.HashMap; import java.util.Map; -public class TestErasureCodingAction extends TestErasureCodingActionBase { +public class TestErasureCodingAction extends TestErasureCodingMiniCluster { @Test public void testEcActionForFile() throws Exception { @@ -34,13 +34,14 @@ public void testEcActionForFile() throws Exception { assertEquals(null, dfsClient.getErasureCodingPolicy(srcPath)); ErasureCodingAction ecAction = new ErasureCodingAction(); + ecAction.setContext(smartContext); String ecTmpPath = "/ssm/ec_tmp/tmp_file"; Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); ecAction.init(args); - ecAction.execute(); + ecAction.run(); assertTrue(ecAction.getExpectedAfterRun()); // the file is stored in ec with default policy assertEquals(dfsClient.getErasureCodingPolicy(srcPath), ecPolicy); @@ -53,11 +54,12 @@ public void testEcActionForDir() throws Exception { assertEquals(null, dfsClient.getErasureCodingPolicy(srcDirPath)); ErasureCodingAction ecAction = new ErasureCodingAction(); + ecAction.setContext(smartContext); Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcDirPath); args.put(ErasureCodingAction.EC_POLICY_NAME, ecPolicy.getName()); ecAction.init(args); - ecAction.execute(); + ecAction.run(); assertTrue(ecAction.getExpectedAfterRun()); assertEquals(dfsClient.getErasureCodingPolicy(srcDirPath), ecPolicy); diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java deleted file mode 100644 index 94915d0aba6..00000000000 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingActionBase.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.smartdata.hdfs.action; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.junit.Before; -import org.smartdata.hdfs.MiniClusterHarness; - -import java.io.IOException; - -public class TestErasureCodingActionBase extends MiniClusterHarness { - ErasureCodingPolicy ecPolicy; - - @Before - public void init() throws Exception { - // use ErasureCodeConstants.XOR_2_1_SCHEMA - ecPolicy = SystemErasureCodingPolicies.getPolicies().get(3); - cluster = new MiniDFSCluster.Builder(smartContext.getConf()). - numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()). - build(); - } - - public void createTestFile(String srcPath, long length) throws IOException { - DFSTestUtil.createFile(dfs, new Path(srcPath), length, (short) 3, 0L); - } -} diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingMiniCluster.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingMiniCluster.java new file mode 100644 index 00000000000..87daf8ee54a --- /dev/null +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingMiniCluster.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.hdfs.action; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.*; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; +import org.junit.After; +import org.junit.Before; +import org.smartdata.SmartContext; +import org.smartdata.conf.SmartConf; +import org.smartdata.conf.SmartConfKeys; +import org.smartdata.hdfs.MiniClusterHarness; + +import java.io.IOException; + +public class TestErasureCodingMiniCluster { + protected ErasureCodingPolicy ecPolicy; + // use the default one, not the one in MiniClusterHarness + public static final long BLOCK_SIZE = DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; + protected MiniDFSCluster cluster; + protected DistributedFileSystem dfs; + protected DFSClient dfsClient; + protected SmartContext smartContext; + + @Before + public void init() throws Exception { + SmartConf conf = new SmartConf(); +// super.initConf(conf); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT); + conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT); + // use ErasureCodeConstants.XOR_2_1_SCHEMA + ecPolicy = SystemErasureCodingPolicies.getPolicies().get(3); + cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()). + build(); + // Add namenode URL to smartContext + conf.set(SmartConfKeys.SMART_DFS_NAMENODE_RPCSERVER_KEY, + "hdfs://" + cluster.getNameNode().getNameNodeAddressHostPortString()); + smartContext = new SmartContext(conf); + cluster.waitActive(); + dfs = cluster.getFileSystem(); + dfsClient = dfs.getClient(); + dfsClient.enableErasureCodingPolicy(ecPolicy.getName()); + } + + public void createTestFile(String srcPath, long length) throws IOException { +// DFSTestUtil.createFile(dfs, new Path(srcPath), length, (short) 3, 0L); + DFSTestUtil.createFile(dfs, new Path(srcPath), 1024, length, BLOCK_SIZE, (short) 3, 0L); + } + + @After + public void shutdown() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } +} diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java index 336a7904ef5..9e563a0ce6d 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -24,10 +24,10 @@ import java.util.HashMap; import java.util.Map; -public class TestUnErasureCodingAction extends TestErasureCodingActionBase { +public class TestUnErasureCodingAction extends TestErasureCodingMiniCluster { @Test - public void testExecute() throws Exception { + public void testUnEcActionForFile() throws Exception { String testDir = "/test_dir"; dfs.mkdirs(new Path(testDir)); dfs.setErasureCodingPolicy(new Path(testDir), ecPolicy.getName()); @@ -36,29 +36,31 @@ public void testExecute() throws Exception { createTestFile(srcPath, 1000); assertEquals(dfsClient.getErasureCodingPolicy(srcPath), ecPolicy); - UnErasureCodingAction unecAction = new UnErasureCodingAction(); + UnErasureCodingAction unEcAction = new UnErasureCodingAction(); + unEcAction.setContext(smartContext); String ecTmpPath = "/ssm/ec_tmp/tmp_file"; Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, srcPath); args.put(ErasureCodingBase.EC_TMP, ecTmpPath); - unecAction.init(args); - unecAction.execute(); - assertTrue(unecAction.getExpectedAfterRun()); + unEcAction.init(args); + unEcAction.run(); + assertTrue(unEcAction.getExpectedAfterRun()); assertNull(dfsClient.getErasureCodingPolicy(srcPath)); } @Test - public void testEcActionForDir() throws Exception { + public void testUnEcActionForDir() throws Exception { String testDir = "/test_dir"; dfs.mkdirs(new Path(testDir)); dfs.setErasureCodingPolicy(new Path(testDir), ecPolicy.getName()); assertEquals(dfsClient.getErasureCodingPolicy(testDir), ecPolicy); - UnErasureCodingAction unecAction = new UnErasureCodingAction(); + UnErasureCodingAction unEcAction = new UnErasureCodingAction(); + unEcAction.setContext(smartContext); Map args = new HashMap<>(); args.put(HdfsAction.FILE_PATH, testDir); - unecAction.init(args); - unecAction.execute(); + unEcAction.init(args); + unEcAction.run(); assertNull(dfs.getErasureCodingPolicy(new Path(testDir))); // create test file, its EC policy is expected to be replication From dee134d2fd7ba719aef4503dad9ec7e955f9cf17 Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 14 Sep 2018 11:25:23 +0800 Subject: [PATCH 19/20] small refine --- .../org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index bf17657efa4..3e359408bde 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -111,6 +111,7 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { return ScheduleResult.SUCCESS; } + // check file lock merely for ec & unec action if (fileLock.contains(srcPath)) { return ScheduleResult.FAIL; } @@ -132,7 +133,6 @@ public ScheduleResult onSchedule(ActionInfo actionInfo, LaunchAction action) { } private String createTmpName(LaunchAction action) { - // need update to DB String path = action.getArgs().get(HdfsAction.FILE_PATH); String fileName; int index = path.lastIndexOf("/"); @@ -151,6 +151,7 @@ private String createTmpName(LaunchAction action) { return tmpName; } + @Override public void onActionFinished(ActionInfo actionInfo) { if (actionInfo.getActionName().equals(ecActionID) || actionInfo.getActionName().equals(unecActionID)) { From 6185cc349687ecd59672afc995b45a5d15c41279 Mon Sep 17 00:00:00 2001 From: philo Date: Fri, 14 Sep 2018 13:12:49 +0800 Subject: [PATCH 20/20] reformat code --- .../java/org/smartdata/hdfs/action/ErasureCodingBase.java | 2 +- .../org/smartdata/hdfs/action/TestErasureCodingAction.java | 4 ++-- .../org/smartdata/hdfs/action/TestUnErasureCodingAction.java | 4 ++-- .../org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java index 91ecdce9a1b..1476de4e44c 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/main/java/org/smartdata/hdfs/action/ErasureCodingBase.java @@ -106,7 +106,7 @@ protected void setAttributes(String src, HdfsFileStatus fileStatus, String dest) dfsClient.setAcl(dest, dfsClient.getAclStatus(src).getEntries()); } //TODO: check ec related record to avoid paradox - for(Map.Entry entry : dfsClient.getXAttrs(src).entrySet()) { + for (Map.Entry entry : dfsClient.getXAttrs(src).entrySet()) { dfsClient.setXAttr(dest, entry.getKey(), entry.getValue(), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java index baeb560d20a..928f5e01ff5 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestErasureCodingAction.java @@ -27,7 +27,7 @@ public class TestErasureCodingAction extends TestErasureCodingMiniCluster { @Test - public void testEcActionForFile() throws Exception { + public void testEcActionForFile() throws Exception { String srcPath = "/ec/test_file"; createTestFile(srcPath, 1000); // the file is stored in replication @@ -48,7 +48,7 @@ public void testEcActionForFile() throws Exception { } @Test - public void testEcActionForDir() throws Exception { + public void testEcActionForDir() throws Exception { String srcDirPath = "/test_dir/"; dfs.mkdirs(new Path(srcDirPath)); assertEquals(null, dfsClient.getErasureCodingPolicy(srcDirPath)); diff --git a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java index 9e563a0ce6d..a0bb314a5a7 100644 --- a/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java +++ b/smart-hadoop-support/smart-hadoop-3.1/src/test/java/org/smartdata/hdfs/action/TestUnErasureCodingAction.java @@ -27,7 +27,7 @@ public class TestUnErasureCodingAction extends TestErasureCodingMiniCluster { @Test - public void testUnEcActionForFile() throws Exception { + public void testUnEcActionForFile() throws Exception { String testDir = "/test_dir"; dfs.mkdirs(new Path(testDir)); dfs.setErasureCodingPolicy(new Path(testDir), ecPolicy.getName()); @@ -49,7 +49,7 @@ public void testUnEcActionForFile() throws Exception { } @Test - public void testUnEcActionForDir() throws Exception { + public void testUnEcActionForDir() throws Exception { String testDir = "/test_dir"; dfs.mkdirs(new Path(testDir)); dfs.setErasureCodingPolicy(new Path(testDir), ecPolicy.getName()); diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java index 3e359408bde..06d3b673471 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/scheduler/ErasureCodingScheduler.java @@ -54,6 +54,7 @@ public ErasureCodingScheduler(SmartContext context, MetaStore metaStore) { this.conf = context.getConf(); this.metaStore = metaStore; } + public List getSupportedActions() { return actions; } @@ -86,7 +87,7 @@ public boolean onSubmit(ActionInfo actionInfo) throws IOException { return true; } - public static boolean isECSupported () { + public static boolean isECSupported() { String[] parts = VersionInfo.getVersion().split("\\."); return Integer.parseInt(parts[0]) == 3; }