From 4b867cf9edf25e70e39194951692133d4253c927 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Mon, 3 Oct 2022 17:49:07 +0800 Subject: [PATCH 1/9] [Bug][Connector-V2] Fix the bug of incorrect path in windows environment --- .../file/sink/writer/AbstractWriteStrategy.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java index 682f660d8e8..5bc8cea14c7 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java @@ -39,7 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -237,7 +236,7 @@ public void beginTransaction(Long checkpointId) { */ public List getTransactionIdFromStates(List fileStates) { String[] pathSegments = new String[]{textFileSinkConfig.getPath(), Constant.SEATUNNEL, jobId}; - String jobDir = String.join(File.separator, pathSegments) + "/"; + String jobDir = String.join("/", pathSegments) + "/"; try { List transactionDirList = FileSystemUtils.dirList(jobDir).stream().map(Path::toString).collect(Collectors.toList()); return transactionDirList.stream().map(dir -> dir.replaceAll(jobDir, "")).collect(Collectors.toList()); @@ -267,7 +266,7 @@ public List snapshotState(long checkpointId) { */ private String getTransactionDir(@NonNull String transactionId) { String[] strings = new String[]{textFileSinkConfig.getTmpPath(), Constant.SEATUNNEL, jobId, transactionId}; - return String.join(File.separator, strings); + return String.join("/", strings); } public String getOrCreateFilePathBeingWritten(@NonNull SeaTunnelRow seaTunnelRow) { @@ -279,7 +278,7 @@ public String getOrCreateFilePathBeingWritten(@NonNull SeaTunnelRow seaTunnelRow return beingWrittenFilePath; } else { String[] pathSegments = new String[]{transactionDirectory, beingWrittenFileKey, generateFileName(transactionId)}; - String newBeingWrittenFilePath = String.join(File.separator, pathSegments); + String newBeingWrittenFilePath = String.join("/", pathSegments); beingWrittenFile.put(beingWrittenFileKey, newBeingWrittenFilePath); if (!Constant.NON_PARTITION.equals(dataPartitionDirAndValuesMap.keySet().toArray()[0].toString())){ partitionDirAndValuesMap.putAll(dataPartitionDirAndValuesMap); @@ -290,6 +289,6 @@ public String getOrCreateFilePathBeingWritten(@NonNull SeaTunnelRow seaTunnelRow public String getTargetLocation(@NonNull String seaTunnelFilePath) { String tmpPath = seaTunnelFilePath.replaceAll(transactionDirectory, textFileSinkConfig.getPath()); - return tmpPath.replaceAll(Constant.NON_PARTITION + File.separator, ""); + return tmpPath.replaceAll(Constant.NON_PARTITION + "/", ""); } } From 2f19a5654b998617aed4fad9c5b173a03aab60e6 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Tue, 4 Oct 2022 20:04:30 +0800 Subject: [PATCH 2/9] [Bug][Connector-V2] Add unit test for FileSystemUtils --- .../file/writer/FileSystemUtilsTest.java | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java new file mode 100644 index 00000000000..4260fc09fd6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.file.writer; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.List; + +public class FileSystemUtilsTest { + private final Configuration conf = new Configuration(); + private final URL resource = FileSystemUtils.class.getResource("/test.orc"); + + @BeforeEach + void init() { + FileSystemUtils.CONF = conf; + } + + @Test + void testGetFileSystem() throws IOException { + assert resource != null; + FileSystem fileSystem = FileSystemUtils.getFileSystem(resource.getPath()); + fileSystem.close(); + } + + @Test + void testGetOutputString() throws IOException { + assert resource != null; + FSDataOutputStream outputStream = FileSystemUtils.getOutputStream(resource.getPath()); + outputStream.close(); + } + + @Test + void testCreateAndDeleteFile() throws IOException { + assert resource != null; + String newFileName = "test.file"; + String path = resource.getPath(); + String parentPath = new File(path).getParent(); + String newFilePath = String.join(File.separator, parentPath, newFileName); + FileSystemUtils.createFile(newFilePath); + Assertions.assertTrue(FileSystemUtils.fileExist(newFilePath)); + FileSystemUtils.deleteFile(newFilePath); + Assertions.assertFalse(FileSystemUtils.fileExist(newFilePath)); + } + + @Test + void testRenameFile() throws IOException { + assert resource != null; + String oldFileName = "test.file"; + String newFileName = "test.new.file"; + String path = resource.getPath(); + String parentPath = new File(path).getParent(); + String oldFilePath = String.join(File.separator, parentPath, oldFileName); + String newFilePath = String.join(File.separator, parentPath, newFileName); + FileSystemUtils.createFile(oldFilePath); + FileSystemUtils.renameFile(oldFilePath, newFilePath, true); + Assertions.assertTrue(FileSystemUtils.fileExist(newFilePath)); + FileSystemUtils.deleteFile(newFilePath); + } + + @Test + void testCreateDirectory() throws IOException { + assert resource != null; + String newDirectoryName = "testDir"; + String path = resource.getPath(); + String parentPath = new File(path).getParent(); + String newDirPath = String.join(File.separator, parentPath, newDirectoryName); + FileSystemUtils.createDir(newDirPath); + Assertions.assertTrue(FileSystemUtils.fileExist(newDirPath)); + FileSystemUtils.deleteFile(newDirPath); + } + + @Test + void testDirList() throws IOException { + assert resource != null; + String path = resource.getPath(); + String parentPath = new File(path).getParent(); + List paths = FileSystemUtils.dirList(parentPath); + Assertions.assertNotNull(paths); + Assertions.assertNotEquals(paths.size(), 0); + } +} From e1973ccff25762550733ad2d25c3fb39cca61a88 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Tue, 4 Oct 2022 20:09:50 +0800 Subject: [PATCH 3/9] [Bug][Connector-V2] Optimize file system utils --- .../seatunnel/file/sink/util/FileSystemUtils.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/FileSystemUtils.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/FileSystemUtils.java index a788d622209..718e42a6737 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/FileSystemUtils.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/util/FileSystemUtils.java @@ -26,9 +26,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -40,7 +40,7 @@ public class FileSystemUtils { public static Configuration CONF; public static FileSystem getFileSystem(@NonNull String path) throws IOException { - FileSystem fileSystem = FileSystem.get(URI.create(path), CONF); + FileSystem fileSystem = FileSystem.get(new File(path).toURI(), CONF); fileSystem.setWriteChecksum(false); return fileSystem; } @@ -89,8 +89,8 @@ public static void renameFile(@NonNull String oldName, @NonNull String newName, LOGGER.info("Delete already file: {}", newPath); } } - if (!fileExist(newName.substring(0, newName.lastIndexOf("/")))) { - createDir(newName.substring(0, newName.lastIndexOf("/"))); + if (!fileExist(newPath.getParent().toString())) { + createDir(newPath.getParent().toString()); } if (fileSystem.rename(oldPath, newPath)) { @@ -119,7 +119,7 @@ public static boolean fileExist(@NonNull String filePath) throws IOException { */ public static List dirList(@NonNull String filePath) throws FileNotFoundException, IOException { FileSystem fileSystem = getFileSystem(filePath); - List pathList = new ArrayList(); + List pathList = new ArrayList<>(); Path fileName = new Path(filePath); FileStatus[] status = fileSystem.listStatus(fileName); if (status != null && status.length > 0) { From 9087f587ae82934a74c708960e7b608c397fa861 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Tue, 4 Oct 2022 20:10:44 +0800 Subject: [PATCH 4/9] [Bug][Connector-V2] Replace '/' to 'File.separator' --- .../file/sink/writer/AbstractWriteStrategy.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java index 5bc8cea14c7..54932735b0d 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/AbstractWriteStrategy.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.regex.Matcher; import java.util.stream.Collectors; public abstract class AbstractWriteStrategy implements WriteStrategy { @@ -141,7 +143,7 @@ public Map> generatorPartitionDir(SeaTunnelRow seaTunnelRow stringBuilder.append(partitionFieldList.get(i)) .append("=") .append(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)]) - .append("/"); + .append(File.separator); vals.add(seaTunnelRow.getFields()[partitionFieldsIndexInRow.get(i)].toString()); } partitionDir = stringBuilder.toString(); @@ -236,7 +238,7 @@ public void beginTransaction(Long checkpointId) { */ public List getTransactionIdFromStates(List fileStates) { String[] pathSegments = new String[]{textFileSinkConfig.getPath(), Constant.SEATUNNEL, jobId}; - String jobDir = String.join("/", pathSegments) + "/"; + String jobDir = String.join(File.separator, pathSegments) + File.separator; try { List transactionDirList = FileSystemUtils.dirList(jobDir).stream().map(Path::toString).collect(Collectors.toList()); return transactionDirList.stream().map(dir -> dir.replaceAll(jobDir, "")).collect(Collectors.toList()); @@ -266,7 +268,7 @@ public List snapshotState(long checkpointId) { */ private String getTransactionDir(@NonNull String transactionId) { String[] strings = new String[]{textFileSinkConfig.getTmpPath(), Constant.SEATUNNEL, jobId, transactionId}; - return String.join("/", strings); + return String.join(File.separator, strings); } public String getOrCreateFilePathBeingWritten(@NonNull SeaTunnelRow seaTunnelRow) { @@ -278,7 +280,7 @@ public String getOrCreateFilePathBeingWritten(@NonNull SeaTunnelRow seaTunnelRow return beingWrittenFilePath; } else { String[] pathSegments = new String[]{transactionDirectory, beingWrittenFileKey, generateFileName(transactionId)}; - String newBeingWrittenFilePath = String.join("/", pathSegments); + String newBeingWrittenFilePath = String.join(File.separator, pathSegments); beingWrittenFile.put(beingWrittenFileKey, newBeingWrittenFilePath); if (!Constant.NON_PARTITION.equals(dataPartitionDirAndValuesMap.keySet().toArray()[0].toString())){ partitionDirAndValuesMap.putAll(dataPartitionDirAndValuesMap); @@ -288,7 +290,8 @@ public String getOrCreateFilePathBeingWritten(@NonNull SeaTunnelRow seaTunnelRow } public String getTargetLocation(@NonNull String seaTunnelFilePath) { - String tmpPath = seaTunnelFilePath.replaceAll(transactionDirectory, textFileSinkConfig.getPath()); - return tmpPath.replaceAll(Constant.NON_PARTITION + "/", ""); + String tmpPath = seaTunnelFilePath.replaceAll(Matcher.quoteReplacement(transactionDirectory), + Matcher.quoteReplacement(textFileSinkConfig.getPath())); + return tmpPath.replaceAll(Constant.NON_PARTITION + Matcher.quoteReplacement(File.separator), ""); } } From 7caa65f144d39730a5b25d38fb7b28cfbcb63ac2 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Tue, 4 Oct 2022 20:13:46 +0800 Subject: [PATCH 5/9] [Bug][Connector-V2] Fix code style --- .../seatunnel/file/writer/FileSystemUtilsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java index 4260fc09fd6..b96e16fc4eb 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java @@ -17,12 +17,12 @@ package org.apache.seatunnel.connectors.seatunnel.file.writer; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From 1cb436aeedea4ccd7b4d812534a687597e7edba2 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Wed, 5 Oct 2022 14:41:24 +0800 Subject: [PATCH 6/9] [Bug][Connector-V2] Fix test cases --- .../connectors/seatunnel/file/writer/FileSystemUtilsTest.java | 2 +- .../connector-file-base/src/test/resources/test.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java index b96e16fc4eb..aa0ce4d3775 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java @@ -34,7 +34,7 @@ public class FileSystemUtilsTest { private final Configuration conf = new Configuration(); - private final URL resource = FileSystemUtils.class.getResource("/test.orc"); + private final URL resource = FileSystemUtils.class.getResource("/test.txt"); @BeforeEach void init() { diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt new file mode 100644 index 00000000000..c1484ddfafe --- /dev/null +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt @@ -0,0 +1 @@ +This is the file that service for file system utils test. \ No newline at end of file From 289c370443ed20dc478b0bdb7248025cc88d7357 Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Wed, 5 Oct 2022 15:38:06 +0800 Subject: [PATCH 7/9] [Improve][Connector-V2] Fix test case --- .../file/writer/FileSystemUtilsTest.java | 2 +- .../src/test/resources/test.txt | 1 - .../src/test/resources/test_filesystem.orc | Bin 0 -> 1959 bytes 3 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt create mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem.orc diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java index aa0ce4d3775..3b48cff04fa 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java @@ -34,7 +34,7 @@ public class FileSystemUtilsTest { private final Configuration conf = new Configuration(); - private final URL resource = FileSystemUtils.class.getResource("/test.txt"); + private final URL resource = FileSystemUtils.class.getResource("/test_filesystem.orc"); @BeforeEach void init() { diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt deleted file mode 100644 index c1484ddfafe..00000000000 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test.txt +++ /dev/null @@ -1 +0,0 @@ -This is the file that service for file system utils test. \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem.orc b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem.orc new file mode 100644 index 0000000000000000000000000000000000000000..5f5d5b969725afa67a7a33b7d858dc9d699eced1 GIT binary patch literal 1959 zcma))dr(u?7RGmSl5lv15ENo~NJzp%OR#{o73B@0K*WebE3X6sWP}h&Kv3p3LO>Lv z$c5CTEdu32MFpl(MU9YB>=kf$UxcDi9+h0MP>QWWdEApQ(^~5v_spCH8o($b4n`BuB=lQ!3;{#J zyhX;1KX00F$g~4kw1dnu;kYeijFycXhl-KE&_f81T1f5Cl2=8Ffc; z`zAPCfTH-}{i!@gx^S;hD0DeYA^GTO^8+-*w3gV3_orhSqs1MSZOY=$`lcQ=4%kpa z>m;E70vaRMk6LUCg1yVQQ2eM(IaSz}MR`Ba+ggkQ-WsDRErCS{2S(k*KwEL2QaNCO zrO<9k0uUD(!VotYJl@~V7T*kTz!edTCpobIq8Q`-o!rkscGDkcl$L`EGzmB%!`-|w zTs(~04a3i`qY20x0?d(!{zgRmbAn#)Fdpgf*-B*`<*%pF{Q{}?W}@b3NPW$W4&iO- z|6mLju2fYWpvaVMPeOGokPH?=^!=bEG7(}wQ}_vi=CpOefO5w6i| zw#i#F!tOM9Pnx+r8UTvUuivTq_MGgiR{3BPtDY~YaJ2n?JYd&Qza!ePsW)n`dL`fO zDX)3|%kIyp1&gKwark|YzbeAh9mjbOR=wj-%>5WyQ2kq0cjZ-c(R2T&zgQJi&xI^D zGw+X|Hf?cAQ=PGF96SE8XQ$Nwb-l&!@Il3KR<|JM$hCTI^z7qNk6B^-m6o!LyEted zz2wDg_1u&vzh%%}(<*XJ`^dm6vGBxrw(qA+HjKzSBHmTM0D&UrLxG{w57<4DWA&Vf z3`}(3S?ld~yyex^md5^FuV%`j=XH7aR9(q&l9y>o=zm(g!*)1Mb!$C@=SFcEO%vu@ zF!uJst>HbRS4%t1@8Yr`+Rkmfh`Jax`Deo-Nv>;REcI@FrFw=%h7F1u&nBo&MLy~N zWHEgDrk}HAjc{pKtvJZYE$$S&zY!Ol#f2J`V}=1NS@Ze(f?r>zhO6R8W;~r zvB8`(Rt+Q7%dfJ2Upl3(Uf!dtx2?;}4p;YzRMnP4!+VBD$HmKQ-@Bo& zJ#Jm5nM-ATy$ixx+GgUxFMqydH#`~Yk=!y_QZhN?yY5l?pM_kTQ1-%4(L@Z8vN|vH z@7Q$X^MN~Q(H9b69_|SJaK(8nS&AuD)`n9|^XYDa{zemrV1M5Effmmd;GyzR$*?Ud z9c$C1;S;m2N93Dj%v5GqyHr^J4RMU3&Ze&ot?k>r>}^Y1@hYcAdOO~0w@6$_uNj%W zH|=%0Y?^od=IujexSyQc)At`G_YPlAQw(-o)xBbOO>tYst24T2wv=A>-4VH;1^rbj zH>ty>{b=bzlv%HoQ7EQD5oLTi=rv{h% z7ofoUi5zaEF&D~w8j@^%(UDoy%&WB`w2Sp-`c0DcM{XawnG>JA^xQWGY%OfY5F;AN z@tH2SZF3Yc4?mE|;;|dw`SU)H&7R2J<;77kQ z>jykkRW*5+U%X#(u1k<{XODAq`Ft&5RlFE{y)#+%;J*Q7f^`d*g!51)v*K)oz0t$Y zvh`m_YQ{Xqq_yVm^V?CrsFmd)Z0m~S$jL3oT-w@CJ?SV=&*AHPTlPGgn{-ewJ_~lg zGG8VC)Ty^^xN6HjjRNWrX literal 0 HcmV?d00001 From bcd598b24ebe23692f2bc18569e3466ffc11807f Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Wed, 5 Oct 2022 16:35:29 +0800 Subject: [PATCH 8/9] [Bug][Connector-V2] Fix test cases --- .../seatunnel/file/writer/FileSystemUtilsTest.java | 2 +- ...est_filesystem.orc => test_filesystem_utils.orc} | Bin 2 files changed, 1 insertion(+), 1 deletion(-) rename seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/{test_filesystem.orc => test_filesystem_utils.orc} (100%) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java index 3b48cff04fa..27d3f1d1eed 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java @@ -34,7 +34,7 @@ public class FileSystemUtilsTest { private final Configuration conf = new Configuration(); - private final URL resource = FileSystemUtils.class.getResource("/test_filesystem.orc"); + private final URL resource = FileSystemUtils.class.getResource("/test_filesystem_utils.orc"); @BeforeEach void init() { diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem.orc b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem_utils.orc similarity index 100% rename from seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem.orc rename to seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem_utils.orc From a348d63d7d4d775dbf63e30e9786e02f0ed3e57c Mon Sep 17 00:00:00 2001 From: tyrantlucifer Date: Wed, 5 Oct 2022 17:17:29 +0800 Subject: [PATCH 9/9] [Bug][Connector-V2] Remove test case --- .../file/writer/FileSystemUtilsTest.java | 107 ------------------ .../test/resources/test_filesystem_utils.orc | Bin 1959 -> 0 bytes 2 files changed, 107 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java delete mode 100644 seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem_utils.orc diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java deleted file mode 100644 index 27d3f1d1eed..00000000000 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/FileSystemUtilsTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.file.writer; - -import org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.List; - -public class FileSystemUtilsTest { - private final Configuration conf = new Configuration(); - private final URL resource = FileSystemUtils.class.getResource("/test_filesystem_utils.orc"); - - @BeforeEach - void init() { - FileSystemUtils.CONF = conf; - } - - @Test - void testGetFileSystem() throws IOException { - assert resource != null; - FileSystem fileSystem = FileSystemUtils.getFileSystem(resource.getPath()); - fileSystem.close(); - } - - @Test - void testGetOutputString() throws IOException { - assert resource != null; - FSDataOutputStream outputStream = FileSystemUtils.getOutputStream(resource.getPath()); - outputStream.close(); - } - - @Test - void testCreateAndDeleteFile() throws IOException { - assert resource != null; - String newFileName = "test.file"; - String path = resource.getPath(); - String parentPath = new File(path).getParent(); - String newFilePath = String.join(File.separator, parentPath, newFileName); - FileSystemUtils.createFile(newFilePath); - Assertions.assertTrue(FileSystemUtils.fileExist(newFilePath)); - FileSystemUtils.deleteFile(newFilePath); - Assertions.assertFalse(FileSystemUtils.fileExist(newFilePath)); - } - - @Test - void testRenameFile() throws IOException { - assert resource != null; - String oldFileName = "test.file"; - String newFileName = "test.new.file"; - String path = resource.getPath(); - String parentPath = new File(path).getParent(); - String oldFilePath = String.join(File.separator, parentPath, oldFileName); - String newFilePath = String.join(File.separator, parentPath, newFileName); - FileSystemUtils.createFile(oldFilePath); - FileSystemUtils.renameFile(oldFilePath, newFilePath, true); - Assertions.assertTrue(FileSystemUtils.fileExist(newFilePath)); - FileSystemUtils.deleteFile(newFilePath); - } - - @Test - void testCreateDirectory() throws IOException { - assert resource != null; - String newDirectoryName = "testDir"; - String path = resource.getPath(); - String parentPath = new File(path).getParent(); - String newDirPath = String.join(File.separator, parentPath, newDirectoryName); - FileSystemUtils.createDir(newDirPath); - Assertions.assertTrue(FileSystemUtils.fileExist(newDirPath)); - FileSystemUtils.deleteFile(newDirPath); - } - - @Test - void testDirList() throws IOException { - assert resource != null; - String path = resource.getPath(); - String parentPath = new File(path).getParent(); - List paths = FileSystemUtils.dirList(parentPath); - Assertions.assertNotNull(paths); - Assertions.assertNotEquals(paths.size(), 0); - } -} diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem_utils.orc b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/test_filesystem_utils.orc deleted file mode 100644 index 5f5d5b969725afa67a7a33b7d858dc9d699eced1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1959 zcma))dr(u?7RGmSl5lv15ENo~NJzp%OR#{o73B@0K*WebE3X6sWP}h&Kv3p3LO>Lv z$c5CTEdu32MFpl(MU9YB>=kf$UxcDi9+h0MP>QWWdEApQ(^~5v_spCH8o($b4n`BuB=lQ!3;{#J zyhX;1KX00F$g~4kw1dnu;kYeijFycXhl-KE&_f81T1f5Cl2=8Ffc; z`zAPCfTH-}{i!@gx^S;hD0DeYA^GTO^8+-*w3gV3_orhSqs1MSZOY=$`lcQ=4%kpa z>m;E70vaRMk6LUCg1yVQQ2eM(IaSz}MR`Ba+ggkQ-WsDRErCS{2S(k*KwEL2QaNCO zrO<9k0uUD(!VotYJl@~V7T*kTz!edTCpobIq8Q`-o!rkscGDkcl$L`EGzmB%!`-|w zTs(~04a3i`qY20x0?d(!{zgRmbAn#)Fdpgf*-B*`<*%pF{Q{}?W}@b3NPW$W4&iO- z|6mLju2fYWpvaVMPeOGokPH?=^!=bEG7(}wQ}_vi=CpOefO5w6i| zw#i#F!tOM9Pnx+r8UTvUuivTq_MGgiR{3BPtDY~YaJ2n?JYd&Qza!ePsW)n`dL`fO zDX)3|%kIyp1&gKwark|YzbeAh9mjbOR=wj-%>5WyQ2kq0cjZ-c(R2T&zgQJi&xI^D zGw+X|Hf?cAQ=PGF96SE8XQ$Nwb-l&!@Il3KR<|JM$hCTI^z7qNk6B^-m6o!LyEted zz2wDg_1u&vzh%%}(<*XJ`^dm6vGBxrw(qA+HjKzSBHmTM0D&UrLxG{w57<4DWA&Vf z3`}(3S?ld~yyex^md5^FuV%`j=XH7aR9(q&l9y>o=zm(g!*)1Mb!$C@=SFcEO%vu@ zF!uJst>HbRS4%t1@8Yr`+Rkmfh`Jax`Deo-Nv>;REcI@FrFw=%h7F1u&nBo&MLy~N zWHEgDrk}HAjc{pKtvJZYE$$S&zY!Ol#f2J`V}=1NS@Ze(f?r>zhO6R8W;~r zvB8`(Rt+Q7%dfJ2Upl3(Uf!dtx2?;}4p;YzRMnP4!+VBD$HmKQ-@Bo& zJ#Jm5nM-ATy$ixx+GgUxFMqydH#`~Yk=!y_QZhN?yY5l?pM_kTQ1-%4(L@Z8vN|vH z@7Q$X^MN~Q(H9b69_|SJaK(8nS&AuD)`n9|^XYDa{zemrV1M5Effmmd;GyzR$*?Ud z9c$C1;S;m2N93Dj%v5GqyHr^J4RMU3&Ze&ot?k>r>}^Y1@hYcAdOO~0w@6$_uNj%W zH|=%0Y?^od=IujexSyQc)At`G_YPlAQw(-o)xBbOO>tYst24T2wv=A>-4VH;1^rbj zH>ty>{b=bzlv%HoQ7EQD5oLTi=rv{h% z7ofoUi5zaEF&D~w8j@^%(UDoy%&WB`w2Sp-`c0DcM{XawnG>JA^xQWGY%OfY5F;AN z@tH2SZF3Yc4?mE|;;|dw`SU)H&7R2J<;77kQ z>jykkRW*5+U%X#(u1k<{XODAq`Ft&5RlFE{y)#+%;J*Q7f^`d*g!51)v*K)oz0t$Y zvh`m_YQ{Xqq_yVm^V?CrsFmd)Z0m~S$jL3oT-w@CJ?SV=&*AHPTlPGgn{-ewJ_~lg zGG8VC)Ty^^xN6HjjRNWrX