From 3460d4c94bea4b92a8fc21628b026de5f2df5d0f Mon Sep 17 00:00:00 2001 From: JohnNiang Date: Wed, 21 Aug 2024 11:45:49 +0800 Subject: [PATCH 1/2] Add support for restoring from backup root Signed-off-by: JohnNiang --- api-docs/openapi/v3_0/aggregated.json | 142 ++++++++++++------ .../v3_0/apis_console.api_v1alpha1.json | 110 ++++++++++++++ .../run/halo/app/migration/BackupFile.java | 34 +++++ .../halo/app/migration/MigrationEndpoint.java | 81 +++++++--- .../halo/app/migration/MigrationService.java | 16 ++ .../migration/impl/MigrationServiceImpl.java | 55 ++++++- .../extensions/role-template-migration.yaml | 18 ++- .../impl/MigrationServiceImplTest.java | 62 ++++++++ 8 files changed, 444 insertions(+), 74 deletions(-) create mode 100644 application/src/main/java/run/halo/app/migration/BackupFile.java diff --git a/api-docs/openapi/v3_0/aggregated.json b/api-docs/openapi/v3_0/aggregated.json index b2ef456a47..ec15fbc3df 100644 --- a/api-docs/openapi/v3_0/aggregated.json +++ b/api-docs/openapi/v3_0/aggregated.json @@ -5239,55 +5239,6 @@ ] } }, - "/apis/api.console.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": { - "get": { - "operationId": "DownloadBackups", - "parameters": [ - { - "description": "Backup name.", - "in": "path", - "name": "name", - "required": true, - "schema": { - "type": "string" - } - }, - { - "description": "Backup filename.", - "in": "path", - "name": "filename", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": {}, - "tags": [ - "MigrationV1alpha1Console" - ] - } - }, - "/apis/api.console.migration.halo.run/v1alpha1/restorations": { - "post": { - "description": "Restore backup by uploading file or providing download link or backup name.", - "operationId": "RestoreBackup", - "requestBody": { - "content": { - "multipart/form-data": { - "schema": { - "$ref": "#/components/schemas/RestoreRequest" - } - } - }, - "required": true - }, - "responses": {}, - "tags": [ - "MigrationV1alpha1Console" - ] - } - }, "/apis/api.content.halo.run/v1alpha1/categories": { "get": { "description": "Lists categories.", @@ -7411,6 +7362,79 @@ ] } }, + "/apis/console.api.migration.halo.run/v1alpha1/backup-files": { + "get": { + "description": "Get backup files from backup root.", + "operationId": "getBackupFiles", + "responses": { + "default": { + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BackupFile" + } + } + } + }, + "description": "default response" + } + }, + "tags": [ + "MigrationV1alpha1Console" + ] + } + }, + "/apis/console.api.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": { + "get": { + "operationId": "DownloadBackups", + "parameters": [ + { + "description": "Backup name.", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Backup filename.", + "in": "path", + "name": "filename", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": {}, + "tags": [ + "MigrationV1alpha1Console" + ] + } + }, + "/apis/console.api.migration.halo.run/v1alpha1/restorations": { + "post": { + "description": "Restore backup by uploading file or providing download link or backup name.", + "operationId": "RestoreBackup", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/RestoreRequest" + } + } + }, + "required": true + }, + "responses": {}, + "tags": [ + "MigrationV1alpha1Console" + ] + } + }, "/apis/console.api.notification.halo.run/v1alpha1/notifiers/default-email-notifier/verify-connection": { "post": { "description": "Verify email sender config.", @@ -15300,6 +15324,22 @@ } } }, + "BackupFile": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "lastModifiedTime": { + "type": "string", + "format": "date-time" + }, + "size": { + "type": "integer", + "format": "int64" + } + } + }, "BackupList": { "required": [ "first", @@ -20684,6 +20724,10 @@ "file": { "type": "string", "format": "binary" + }, + "filename": { + "type": "string", + "description": "Filename of backup file in backups root." } } }, diff --git a/api-docs/openapi/v3_0/apis_console.api_v1alpha1.json b/api-docs/openapi/v3_0/apis_console.api_v1alpha1.json index 595b83bd13..b91a91f734 100644 --- a/api-docs/openapi/v3_0/apis_console.api_v1alpha1.json +++ b/api-docs/openapi/v3_0/apis_console.api_v1alpha1.json @@ -3106,6 +3106,79 @@ ] } }, + "/apis/console.api.migration.halo.run/v1alpha1/backup-files": { + "get": { + "description": "Get backup files from backup root.", + "operationId": "getBackupFiles", + "responses": { + "default": { + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BackupFile" + } + } + } + }, + "description": "default response" + } + }, + "tags": [ + "MigrationV1alpha1Console" + ] + } + }, + "/apis/console.api.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": { + "get": { + "operationId": "DownloadBackups", + "parameters": [ + { + "description": "Backup name.", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Backup filename.", + "in": "path", + "name": "filename", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": {}, + "tags": [ + "MigrationV1alpha1Console" + ] + } + }, + "/apis/console.api.migration.halo.run/v1alpha1/restorations": { + "post": { + "description": "Restore backup by uploading file or providing download link or backup name.", + "operationId": "RestoreBackup", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/RestoreRequest" + } + } + }, + "required": true + }, + "responses": {}, + "tags": [ + "MigrationV1alpha1Console" + ] + } + }, "/apis/console.api.notification.halo.run/v1alpha1/notifiers/default-email-notifier/verify-connection": { "post": { "description": "Verify email sender config.", @@ -3376,6 +3449,22 @@ } } }, + "BackupFile": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "lastModifiedTime": { + "type": "string", + "format": "date-time" + }, + "size": { + "type": "integer", + "format": "int64" + } + } + }, "Category": { "required": [ "apiVersion", @@ -5405,6 +5494,27 @@ } } }, + "RestoreRequest": { + "type": "object", + "properties": { + "backupName": { + "type": "string", + "description": "Backup metadata name." + }, + "downloadUrl": { + "type": "string", + "description": "Remote backup HTTP URL." + }, + "file": { + "type": "string", + "format": "binary" + }, + "filename": { + "type": "string", + "description": "Filename of backup file in backups root." + } + } + }, "RevertSnapshotForPostParam": { "required": [ "snapshotName" diff --git a/application/src/main/java/run/halo/app/migration/BackupFile.java b/application/src/main/java/run/halo/app/migration/BackupFile.java new file mode 100644 index 0000000000..83fdda72cd --- /dev/null +++ b/application/src/main/java/run/halo/app/migration/BackupFile.java @@ -0,0 +1,34 @@ +package run.halo.app.migration; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import java.nio.file.Path; +import java.time.Instant; +import lombok.Data; + +/** + * Backup file. + * + * @author johnniang + */ +@Data +public class BackupFile { + + @JsonIgnore + private Path path; + + /** + * Filename of backup file. + */ + private String filename; + + /** + * Size of backup file. + */ + private long size; + + /** + * Last modified time of backup file. + */ + private Instant lastModifiedTime; + +} diff --git a/application/src/main/java/run/halo/app/migration/MigrationEndpoint.java b/application/src/main/java/run/halo/app/migration/MigrationEndpoint.java index 83b6d339a9..10abf9851e 100644 --- a/application/src/main/java/run/halo/app/migration/MigrationEndpoint.java +++ b/application/src/main/java/run/halo/app/migration/MigrationEndpoint.java @@ -1,6 +1,7 @@ package run.halo.app.migration; import static io.swagger.v3.oas.annotations.media.Schema.RequiredMode.NOT_REQUIRED; +import static org.springdoc.core.fn.builders.apiresponse.Builder.responseBuilder; import static org.springdoc.core.fn.builders.content.Builder.contentBuilder; import static org.springdoc.core.fn.builders.parameter.Builder.parameterBuilder; import static org.springdoc.core.fn.builders.requestbody.Builder.requestBodyBuilder; @@ -12,10 +13,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.Optional; +import java.util.function.Supplier; import org.springdoc.webflux.core.fn.SpringdocRouteBuilder; import org.springframework.core.io.buffer.DataBuffer; import org.springframework.core.io.buffer.DataBufferUtils; import org.springframework.core.io.buffer.DefaultDataBufferFactory; +import org.springframework.data.util.Optionals; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.codec.multipart.FilePart; @@ -24,7 +27,9 @@ import org.springframework.stereotype.Component; import org.springframework.util.MultiValueMap; import org.springframework.util.StreamUtils; +import org.springframework.util.StringUtils; import org.springframework.web.reactive.function.server.RouterFunction; +import org.springframework.web.reactive.function.server.ServerRequest; import org.springframework.web.reactive.function.server.ServerResponse; import org.springframework.web.server.ServerWebInputException; import reactor.core.publisher.Flux; @@ -55,6 +60,15 @@ public MigrationEndpoint(MigrationService migrationService, public RouterFunction endpoint() { var tag = "MigrationV1alpha1Console"; return SpringdocRouteBuilder.route() + .GET("/backup-files", + this::getBackups, + builder -> builder.operationId("getBackupFiles") + .tag(tag) + .description("Get backup files from backup root.") + .response(responseBuilder() + .implementationArray(BackupFile.class) + ) + ) .GET("/backups/{name}/files/{filename}", request -> { var name = request.pathVariable("name"); @@ -86,7 +100,7 @@ public RouterFunction endpoint() { var content = getContent(restoreRequest) .switchIfEmpty(Mono.error(() -> new ServerWebInputException( "Please upload a file " - + "or provide a download link or backup name."))); + + "or provide a download link or backup name."))); return migrationService.restore(content); }) .then(Mono.defer( @@ -95,7 +109,7 @@ public RouterFunction endpoint() { builder -> builder .tag(tag) .description("Restore backup by uploading file " - + "or providing download link or backup name.") + + "or providing download link or backup name.") .operationId("RestoreBackup") .requestBody(requestBodyBuilder() .required(true) @@ -108,8 +122,22 @@ public RouterFunction endpoint() { .build(); } + private Mono getBackups(ServerRequest request) { + var backupFiles = migrationService.getBackupFiles(); + return ServerResponse.ok().body(backupFiles, BackupFile.class); + } + private Flux getContent(RestoreRequest request) { - var downloadContent = request.getDownloadUrl() + Supplier>> contentFromFilename = () -> + request.getFilename().map(filename -> migrationService.getBackupFile(filename) + .map(BackupFile::getPath) + .flatMapMany( + path -> DataBufferUtils.read( + path, + DefaultDataBufferFactory.sharedInstance, + StreamUtils.BUFFER_SIZE))); + + Supplier>> contentFromDownloadUrl = () -> request.getDownloadUrl() .map(downloadURL -> { try { var url = new URL(downloadURL); @@ -121,23 +149,27 @@ private Flux getContent(RestoreRequest request) { // Should never happen return Flux.error(e); } - }) - .orElseGet(Flux::empty); + }); - var uploadContent = request.getFile() - .map(Part::content) - .orElseGet(Flux::empty); + Supplier>> contentFromUpload = () -> request.getFile() + .map(Part::content); - var backupFileContent = request.getBackupName() + Supplier>> contentFromBackupName = () -> request.getBackupName() .map(backupName -> client.get(Backup.class, backupName) .flatMap(migrationService::download) .flatMapMany(resource -> DataBufferUtils.read(resource, DefaultDataBufferFactory.sharedInstance, - StreamUtils.BUFFER_SIZE))) - .orElseGet(Flux::empty); - return uploadContent - .switchIfEmpty(downloadContent) - .switchIfEmpty(backupFileContent); + StreamUtils.BUFFER_SIZE))); + + return Optionals.firstNonEmpty( + contentFromUpload, + contentFromDownloadUrl, + contentFromBackupName, + contentFromFilename + ) + .orElseGet(() -> Flux.error(new ServerWebInputException(""" + Please upload a file or provide a download link or backup name or backup filename.\ + """))); } @Schema(types = "object") @@ -157,13 +189,26 @@ public Optional getFile() { return Optional.empty(); } + @Schema(requiredMode = NOT_REQUIRED, name = "filename", description = """ + Filename of backup file in backups root.\ + """) + public Optional getFilename() { + var part = multipart.getFirst("filename"); + if (part instanceof FormFieldPart filenamePart) { + return Optional.of(filenamePart.value()) + .filter(StringUtils::hasText); + } + return Optional.empty(); + } + @Schema(requiredMode = NOT_REQUIRED, name = "downloadUrl", description = "Remote backup HTTP URL.") public Optional getDownloadUrl() { var part = multipart.getFirst("downloadUrl"); if (part instanceof FormFieldPart downloadUrlPart) { - return Optional.of(downloadUrlPart.value()); + return Optional.of(downloadUrlPart.value()) + .filter(StringUtils::hasText); } return Optional.empty(); } @@ -174,16 +219,16 @@ public Optional getDownloadUrl() { public Optional getBackupName() { var part = multipart.getFirst("backupName"); if (part instanceof FormFieldPart backupNamePart) { - return Optional.of(backupNamePart.value()); + return Optional.of(backupNamePart.value()) + .filter(StringUtils::hasText); } return Optional.empty(); } } - @Override public GroupVersion groupVersion() { return GroupVersion.parseAPIVersion( - "api.console." + Constant.GROUP + "/" + Constant.VERSION); + "console.api." + Constant.GROUP + "/" + Constant.VERSION); } } diff --git a/application/src/main/java/run/halo/app/migration/MigrationService.java b/application/src/main/java/run/halo/app/migration/MigrationService.java index 5a87cb8326..4639e31be7 100644 --- a/application/src/main/java/run/halo/app/migration/MigrationService.java +++ b/application/src/main/java/run/halo/app/migration/MigrationService.java @@ -3,6 +3,7 @@ import org.reactivestreams.Publisher; import org.springframework.core.io.Resource; import org.springframework.core.io.buffer.DataBuffer; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; public interface MigrationService { @@ -21,4 +22,19 @@ public interface MigrationService { */ Mono cleanup(Backup backup); + /** + * Gets backup files. + * + * @return backup files, sorted by last modified time. + */ + Flux getBackupFiles(); + + /** + * Get backup file by filename. + * + * @param filename filename of backup file + * @return backup file or empty if file is not found + */ + Mono getBackupFile(String filename); + } diff --git a/application/src/main/java/run/halo/app/migration/impl/MigrationServiceImpl.java b/application/src/main/java/run/halo/app/migration/impl/MigrationServiceImpl.java index 9e3a668bab..6d21b50bad 100644 --- a/application/src/main/java/run/halo/app/migration/impl/MigrationServiceImpl.java +++ b/application/src/main/java/run/halo/app/migration/impl/MigrationServiceImpl.java @@ -1,6 +1,8 @@ package run.halo.app.migration.impl; import static java.nio.file.Files.deleteIfExists; +import static java.util.Comparator.comparing; +import static org.apache.commons.io.FilenameUtils.isExtension; import static org.springframework.util.FileSystemUtils.copyRecursively; import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal; import static run.halo.app.infra.utils.FileUtils.copyRecursively; @@ -19,8 +21,10 @@ import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.Set; +import java.util.stream.BaseStream; import lombok.extern.slf4j.Slf4j; import org.reactivestreams.Publisher; +import org.springframework.beans.factory.InitializingBean; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.core.io.buffer.DataBuffer; @@ -40,11 +44,12 @@ import run.halo.app.infra.properties.HaloProperties; import run.halo.app.infra.utils.FileUtils; import run.halo.app.migration.Backup; +import run.halo.app.migration.BackupFile; import run.halo.app.migration.MigrationService; @Slf4j @Service -public class MigrationServiceImpl implements MigrationService { +public class MigrationServiceImpl implements MigrationService, InitializingBean { private final ExtensionStoreRepository repository; @@ -163,6 +168,49 @@ public Mono cleanup(Backup backup) { }).subscribeOn(scheduler); } + @Override + public Flux getBackupFiles() { + return Flux.using( + () -> Files.list(getBackupsRoot()), + Flux::fromStream, + BaseStream::close + ) + .filter(Files::isRegularFile) + .filter(Files::isReadable) + .filter(path -> isExtension(path.getFileName().toString(), "zip")) + .map(this::toBackupFile) + .sort(comparing(BackupFile::getLastModifiedTime).reversed() + .thenComparing(BackupFile::getFilename) + ) + .subscribeOn(this.scheduler); + } + + @Override + public Mono getBackupFile(String filename) { + return Mono.fromCallable(() -> { + var backupsRoot = getBackupsRoot(); + var backupFilePath = backupsRoot.resolve(filename); + checkDirectoryTraversal(backupsRoot, backupFilePath); + if (Files.notExists(backupFilePath)) { + return null; + } + return toBackupFile(backupFilePath); + }).subscribeOn(this.scheduler); + } + + private BackupFile toBackupFile(Path path) { + var backupFile = new BackupFile(); + backupFile.setPath(path); + backupFile.setFilename(path.getFileName().toString()); + try { + backupFile.setSize(Files.size(path)); + backupFile.setLastModifiedTime(Files.getLastModifiedTime(path).toInstant()); + return backupFile; + } catch (IOException e) { + throw Exceptions.propagate(e); + } + } + private Mono restoreWorkdir(Path backupRoot) { return Mono.create(sink -> { try { @@ -264,4 +312,9 @@ private Mono backupExtensions(Path baseDir) { FileUtils::closeQuietly)) .subscribeOn(scheduler); } + + @Override + public void afterPropertiesSet() throws Exception { + Files.createDirectories(getBackupsRoot()); + } } diff --git a/application/src/main/resources/extensions/role-template-migration.yaml b/application/src/main/resources/extensions/role-template-migration.yaml index 860081223f..6edc1c8ffc 100644 --- a/application/src/main/resources/extensions/role-template-migration.yaml +++ b/application/src/main/resources/extensions/role-template-migration.yaml @@ -10,9 +10,15 @@ metadata: rbac.authorization.halo.run/ui-permissions: | ["system:migrations:manage"] rules: - - apiGroups: ["api.console.migration.halo.run"] - resources: ["restorations"] - verbs: ["create"] - - apiGroups: ["migration.halo.run"] - resources: ["backups"] - verbs: ["list", "get", "create", "update", "delete"] + - apiGroups: [ "console.api.migration.halo.run" ] + resources: [ "restorations" ] + verbs: [ "create" ] + - apiGroups: [ "console.api.migration.halo.run" ] + resources: [ "backup-files" ] + verbs: [ "list" ] + - apiGroups: [ "console.api.migration.halo.run" ] + resources: [ "backups/files" ] + verbs: [ "get" ] + - apiGroups: [ "migration.halo.run" ] + resources: [ "backups" ] + verbs: [ "list", "get", "create", "update", "delete", "patch" ] diff --git a/application/src/test/java/run/halo/app/migration/impl/MigrationServiceImplTest.java b/application/src/test/java/run/halo/app/migration/impl/MigrationServiceImplTest.java index 808ae0c656..e515e02c65 100644 --- a/application/src/test/java/run/halo/app/migration/impl/MigrationServiceImplTest.java +++ b/application/src/test/java/run/halo/app/migration/impl/MigrationServiceImplTest.java @@ -14,6 +14,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileTime; +import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.zip.ZipInputStream; @@ -198,6 +200,66 @@ void downloadBackupWhichDoesNotExist() { verify(backupRoot).get(); } + @Test + void getBackupFilesTest() throws Exception { + var now = Instant.now(); + var backup1 = tempDir.resolve("backup1.zip"); + Files.writeString(backup1, "fake-content"); + Files.setLastModifiedTime(backup1, FileTime.from(now)); + + var backup2 = tempDir.resolve("backup2.zip"); + Files.writeString(backup2, "fake--content"); + Files.setLastModifiedTime( + backup2, + FileTime.from(now.plus(Duration.ofSeconds(1))) + ); + + var backup3 = tempDir.resolve("backup3.not-a-zip"); + Files.writeString(backup3, "fake-content"); + Files.setLastModifiedTime( + backup3, + FileTime.from(now.plus(Duration.ofSeconds(2))) + ); + when(backupRoot.get()).thenReturn(tempDir); + + migrationService.afterPropertiesSet(); + migrationService.getBackupFiles() + .as(StepVerifier::create) + .assertNext(backupFile -> { + assertEquals("backup2.zip", backupFile.getFilename()); + assertEquals(13, backupFile.getSize()); + assertEquals(now.plus(Duration.ofSeconds(1)), backupFile.getLastModifiedTime()); + }) + .assertNext(backupFile -> { + assertEquals("backup1.zip", backupFile.getFilename()); + assertEquals(12, backupFile.getSize()); + assertEquals(now, backupFile.getLastModifiedTime()); + }) + .verifyComplete(); + } + + @Test + void getBackupFileTest() throws Exception { + var now = Instant.now(); + Files.writeString(tempDir.resolve("backup.zip"), "fake-content"); + Files.setLastModifiedTime(tempDir.resolve("backup.zip"), FileTime.from(now)); + when(backupRoot.get()).thenReturn(tempDir); + + migrationService.afterPropertiesSet(); + migrationService.getBackupFile("backup.zip") + .as(StepVerifier::create) + .assertNext(backupFile -> { + assertEquals("backup.zip", backupFile.getFilename()); + assertEquals(12, backupFile.getSize()); + assertEquals(now, backupFile.getLastModifiedTime()); + }) + .verifyComplete(); + + migrationService.getBackupFile("backup-not-exist.zip") + .as(StepVerifier::create) + .verifyComplete(); + } + Backup createSucceededBackup(String name, String filename) { var metadata = new Metadata(); metadata.setName(name); From 6cd8dc85550c41f5ffd539f2b984e31c4ea72a91 Mon Sep 17 00:00:00 2001 From: Ryan Wang Date: Wed, 21 Aug 2024 11:46:30 +0800 Subject: [PATCH 2/2] Adapt console for restoring from backup root Signed-off-by: JohnNiang --- .../modules/system/backup/tabs/Restore.vue | 60 ++++++++---- .../api-client/src/.openapi-generator/FILES | 1 + .../src/api/migration-v1alpha1-console-api.ts | 95 +++++++++++++++++-- .../api-client/src/models/backup-file.ts | 42 ++++++++ ui/packages/api-client/src/models/index.ts | 1 + ui/src/locales/en.yaml | 15 ++- ui/src/locales/zh-CN.yaml | 5 +- ui/src/locales/zh-TW.yaml | 5 +- 8 files changed, 189 insertions(+), 35 deletions(-) create mode 100644 ui/packages/api-client/src/models/backup-file.ts diff --git a/ui/console-src/modules/system/backup/tabs/Restore.vue b/ui/console-src/modules/system/backup/tabs/Restore.vue index e671172fa4..ba8a580fc8 100644 --- a/ui/console-src/modules/system/backup/tabs/Restore.vue +++ b/ui/console-src/modules/system/backup/tabs/Restore.vue @@ -1,11 +1,13 @@