support export/import configs

apolloconfig · Oct 31, 2021 · ad86f8b · ad86f8b
1 parent fb135bb
commit ad86f8b
Show file tree

Hide file tree

Showing 36 changed files with 1,718 additions and 311 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -21,6 +21,8 @@ Apollo 2.0.0
 * [Bump version to 2.0.0 and drop java 1.7 support](https://github.com/apolloconfig/apollo/pull/4015)
 * [Optimize home page style](https://github.com/apolloconfig/apollo/pull/4052)
 * [Support Java 17](https://github.com/apolloconfig/apollo/pull/4060)
+* [Support export/import configs by apollo env](https://github.com/apolloconfig/apollo/pull/3947)
+
 
 ------------------
 All issues and pull requests are [here](https://github.com/ctripcorp/apollo/milestone/8?closed=1)
diff --git a/...vice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java b/...vice/src/main/java/com/ctrip/framework/apollo/adminservice/controller/ItemController.java
@@ -30,6 +30,8 @@
 import com.ctrip.framework.apollo.common.exception.BadRequestException;
 import com.ctrip.framework.apollo.common.exception.NotFoundException;
 import com.ctrip.framework.apollo.common.utils.BeanUtils;
+import com.ctrip.framework.apollo.core.utils.StringUtils;
+
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -88,6 +90,33 @@ public ItemDTO create(@PathVariable("appId") String appId,
     return dto;
   }
 
+  @PostMapping("/apps/{appId}/clusters/{clusterName}/namespaces/{namespaceName}/comment_items")
+  public ItemDTO createComment(@PathVariable("appId") String appId,
+                        @PathVariable("clusterName") String clusterName,
+                        @PathVariable("namespaceName") String namespaceName, @RequestBody ItemDTO dto) {
+    if (!StringUtils.isBlank(dto.getKey()) || !StringUtils.isBlank(dto.getValue())) {
+      throw new BadRequestException("Comment item's key or value should be blank.");
+    }
+    if (StringUtils.isBlank(dto.getComment())) {
+      throw new BadRequestException("Comment item's comment should not be blank.");
+    }
+
+    // check if comment existed
+    List<Item> allItems = itemService.findItemsWithOrdered(appId, clusterName, namespaceName);
+    for (Item item : allItems) {
+      if (StringUtils.isBlank(item.getKey()) && StringUtils.isBlank(item.getValue()) &&
+          Objects.equals(item.getComment(), dto.getComment())) {
+        return BeanUtils.transform(ItemDTO.class, item);
+      }
+    }
+
+    Item entity = BeanUtils.transform(Item.class, dto);
+    entity = itemService.saveComment(entity);
+
+    return BeanUtils.transform(ItemDTO.class, entity);
+  }
+
+
   @PreAcquireNamespaceLock
   @PutMapping("/apps/{appId}/clusters/{clusterName}/namespaces/{namespaceName}/items/{itemId}")
   public ItemDTO update(@PathVariable("appId") String appId,
@@ -109,7 +138,7 @@ public ItemDTO update(@PathVariable("appId") String appId,
     Item entity = BeanUtils.transform(Item.class, itemDTO);
 
     ConfigChangeContentBuilder builder = new ConfigChangeContentBuilder();
-   
+
     Item beforeUpdateItem = BeanUtils.transform(Item.class, managedEntity);
 
     //protect. only value,comment,lastModifiedBy can be modified

diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ItemService.java
@@ -26,6 +26,7 @@
 import com.ctrip.framework.apollo.common.exception.NotFoundException;
 import com.ctrip.framework.apollo.common.utils.BeanUtils;
 import com.ctrip.framework.apollo.core.utils.StringUtils;
+
 import org.springframework.context.annotation.Lazy;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
@@ -165,6 +166,26 @@ public Item save(Item entity) {
     return item;
   }
 
+  @Transactional
+  public Item saveComment(Item entity) {
+    entity.setKey("");
+    entity.setValue("");
+    entity.setId(0);//protection
+
+    if (entity.getLineNum() == 0) {
+      Item lastItem = findLastOne(entity.getNamespaceId());
+      int lineNum = lastItem == null ? 1 : lastItem.getLineNum() + 1;
+      entity.setLineNum(lineNum);
+    }
+
+    Item item = itemRepository.save(entity);
+
+    auditService.audit(Item.class.getSimpleName(), item.getId(), Audit.OP.INSERT,
+                       item.getDataChangeCreatedBy());
+
+    return item;
+  }
+
   @Transactional
   public Item update(Item item) {
     checkItemValueLength(item.getNamespaceId(), item.getValue());

diff --git a/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java b/apollo-common/src/main/java/com/ctrip/framework/apollo/common/constants/GsonType.java
@@ -19,6 +19,7 @@
 import com.google.gson.reflect.TypeToken;
 
 import com.ctrip.framework.apollo.common.dto.GrayReleaseRuleItemDTO;
+import com.ctrip.framework.apollo.common.dto.ItemDTO;
 
 import java.lang.reflect.Type;
 import java.util.List;
@@ -30,4 +31,5 @@ public interface GsonType {
 
   Type RULE_ITEMS = new TypeToken<List<GrayReleaseRuleItemDTO>>() {}.getType();
 
+  Type ITEM_DTOS = new TypeToken<List<ItemDTO>>(){}.getType();
 }
diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/api/AdminServiceAPI.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/api/AdminServiceAPI.java
@@ -204,6 +204,11 @@ public ItemDTO createItem(String appId, Env env, String clusterName, String name
           item, ItemDTO.class, appId, clusterName, namespace);
     }
 
+    public ItemDTO createCommentItem(String appId, Env env, String clusterName, String namespace, ItemDTO item) {
+      return restTemplate.post(env, "apps/{appId}/clusters/{clusterName}/namespaces/{namespaceName}/comment_items",
+                               item, ItemDTO.class, appId, clusterName, namespace);
+    }
+
     public void deleteItem(Env env, long itemId, String operator) {
 
       restTemplate.delete(env, "items/{itemId}?operator={operator}", itemId, operator);

diff --git a/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java b/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsExportController.java
@@ -16,21 +16,17 @@
  */
 package com.ctrip.framework.apollo.portal.controller;
 
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
 import com.ctrip.framework.apollo.common.exception.ServiceException;
 import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
 import com.ctrip.framework.apollo.portal.entity.bo.NamespaceBO;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.service.ConfigsExportService;
 import com.ctrip.framework.apollo.portal.service.NamespaceService;
 import com.ctrip.framework.apollo.portal.util.NamespaceBOUtils;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Date;
-import java.util.List;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
+
 import org.apache.commons.lang.time.DateFormatUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,15 +35,26 @@
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
 /**
  * jian.tan
  */
 @RestController
 public class ConfigsExportController {
 
-  private static final Logger logger = LoggerFactory.getLogger(ConfigsExportController.class);
+  private static final Logger logger        = LoggerFactory.getLogger(ConfigsExportController.class);
+  private static final String ENV_SEPARATOR = ",";
 
   private final ConfigsExportService configsExportService;
 
@@ -74,12 +81,17 @@ public ConfigsExportController(
   @PreAuthorize(value = "[email protected](#appId, #env, #namespaceName)")
   @GetMapping("/apps/{appId}/envs/{env}/clusters/{clusterName}/namespaces/{namespaceName}/items/export")
   public void exportItems(@PathVariable String appId, @PathVariable String env,
-      @PathVariable String clusterName, @PathVariable String namespaceName,
-      HttpServletResponse res) {
+                                  @PathVariable String clusterName, @PathVariable String namespaceName,
+                                  HttpServletResponse res) {
     List<String> fileNameSplit = Splitter.on(".").splitToList(namespaceName);
 
-    String fileName = fileNameSplit.size() <= 1 ? Joiner.on(".")
-        .join(namespaceName, ConfigFileFormat.Properties.getValue()) : namespaceName;
+    String fileName = namespaceName;
+
+    //properties file or public namespace has not suffix (.properties)
+    if (fileNameSplit.size() <= 1 || !ConfigFileFormat.isValidFormat(fileNameSplit.get(fileNameSplit.size() - 1))) {
+      fileName = Joiner.on(".").join(namespaceName, ConfigFileFormat.Properties.getValue());
+    }
+
     NamespaceBO namespaceBO = namespaceService.loadNamespaceBO(appId, Env.valueOf
         (env), clusterName, namespaceName);
 
@@ -96,21 +108,26 @@ public void exportItems(@PathVariable String appId, @PathVariable String env,
   }
 
   /**
-   * Export all configs in a compressed file.
-   * Just export namespace which current exists read permission.
-   * The permission check in service.
+   * Export all configs in a compressed file. Just export namespace which current exists read permission. The permission
+   * check in service.
    */
-  @GetMapping("/export")
-  public void exportAll(HttpServletRequest request, HttpServletResponse response) throws IOException {
+  @GetMapping("/configs/export")
+  public void exportAll(@RequestParam(value = "envs") String envs,
+                        HttpServletRequest request, HttpServletResponse response) throws IOException {
     // filename must contain the information of time
     final String filename = "apollo_config_export_" + DateFormatUtils.format(new Date(), "yyyy_MMdd_HH_mm_ss") + ".zip";
     // log who download the configs
-    logger.info("Download configs, remote addr [{}], remote host [{}]. Filename is [{}]", request.getRemoteAddr(), request.getRemoteHost(), filename);
+    logger.info("Download configs, remote addr [{}], remote host [{}]. Filename is [{}]", request.getRemoteAddr(),
+                request.getRemoteHost(), filename);
     // set downloaded filename
     response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + filename);
 
+    List<Env>
+        exportEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
     try (OutputStream outputStream = response.getOutputStream()) {
-      configsExportService.exportAllTo(outputStream);
+      configsExportService.exportData(outputStream, exportEnvs);
     }
   }
 

diff --git a/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java b/...l/src/main/java/com/ctrip/framework/apollo/portal/controller/ConfigsImportController.java
@@ -16,27 +16,38 @@
  */
 package com.ctrip.framework.apollo.portal.controller;
 
+import com.google.common.base.Splitter;
+
 import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
+import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.service.ConfigsImportService;
 import com.ctrip.framework.apollo.portal.util.ConfigFileUtils;
-import java.io.IOException;
+
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.zip.ZipInputStream;
+
 /**
  * Import the configs from file.
  * First version: move code from {@link ConfigsExportController}
  * @author wxq
  */
 @RestController
 public class ConfigsImportController {
+  private static final String ENV_SEPARATOR = ",";
 
   private final ConfigsImportService configsImportService;
 
+
   public ConfigsImportController(
       final ConfigsImportService configsImportService
   ) {
@@ -53,13 +64,43 @@ public ConfigsImportController(
   @PreAuthorize(value = "@permissionValidator.hasModifyNamespacePermission(#appId, #namespaceName, #env)")
   @PostMapping("/apps/{appId}/envs/{env}/clusters/{clusterName}/namespaces/{namespaceName}/items/import")
   public void importConfigFile(@PathVariable String appId, @PathVariable String env,
-      @PathVariable String clusterName, @PathVariable String namespaceName,
-      @RequestParam("file") MultipartFile file) throws IOException {
+                               @PathVariable String clusterName, @PathVariable String namespaceName,
+                               @RequestParam("file") MultipartFile file) throws IOException {
     // check file
     ConfigFileUtils.check(file);
     final String format = ConfigFileUtils.getFormat(file.getOriginalFilename());
-    final String standardFilename = ConfigFileUtils.toFilename(appId, clusterName, namespaceName, 
-            ConfigFileFormat.fromString(format));
-    configsImportService.importOneConfigFromFile(env, standardFilename, file.getInputStream());
+    final String standardFilename = ConfigFileUtils.toFilename(appId, clusterName,
+                                                               namespaceName,
+                                                               ConfigFileFormat.fromString(format));
+
+    configsImportService.forceImportNamespaceFromFile(Env.valueOf(env), standardFilename, file.getInputStream());
+  }
+
+  @PostMapping(value = "/configs/import", params = "conflictAction=cover")
+  public void importConfigByZipWithCoverConflictNamespace(@RequestParam(value = "envs") String envs,
+                                @RequestParam("file") MultipartFile file) throws IOException {
+
+    List<Env>
+        importEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
+    byte[] bytes = file.getBytes();
+    try (ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(bytes))) {
+      configsImportService.importDataFromZipFile(importEnvs, zipInputStream, false);
+    }
+  }
+
+  @PostMapping(value = "/configs/import", params = "conflictAction=ignore")
+  public void importConfigByZipWithIgnoreConflictNamespace(@RequestParam(value = "envs") String envs,
+                                @RequestParam("file") MultipartFile file) throws IOException {
+
+    List<Env>
+        importEnvs =
+        Splitter.on(ENV_SEPARATOR).splitToList(envs).stream().map(env -> Env.valueOf(env)).collect(Collectors.toList());
+
+    byte[] bytes = file.getBytes();
+    try (ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(bytes))) {
+      configsImportService.importDataFromZipFile(importEnvs, zipInputStream, true);
+    }
   }
 }
diff --git a/...portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java b/...portal/src/main/java/com/ctrip/framework/apollo/portal/controller/InstanceController.java
@@ -69,7 +69,7 @@ public ResponseEntity<Number> getInstanceCountByNamespace(@PathVariable String e
                                                               @RequestParam String clusterName,
                                                               @RequestParam String namespaceName) {
 
-        int count = instanceService.getInstanceCountByNamepsace(appId, Env.valueOf(env), clusterName, namespaceName);
+        int count = instanceService.getInstanceCountByNamespace(appId, Env.valueOf(env), clusterName, namespaceName);
         return ResponseEntity.ok(new Number(count));
     }
 

diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ConfigBO.java
@@ -40,22 +40,25 @@ public class ConfigBO {
 
   private final ConfigFileFormat format;
 
+  private final boolean isPublic;
+
   public ConfigBO(Env env, String ownerName, String appId, String clusterName,
-      String namespace, String configFileContent, ConfigFileFormat format) {
+      String namespace, boolean isPublic, String configFileContent, ConfigFileFormat format) {
     this.env = env;
     this.ownerName = ownerName;
     this.appId = appId;
     this.clusterName = clusterName;
     this.namespace = namespace;
+    this.isPublic = isPublic;
     this.configFileContent = configFileContent;
     this.format = format;
   }
 
   public ConfigBO(Env env, String ownerName, String appId, String clusterName, NamespaceBO namespaceBO) {
     this(env, ownerName, appId, clusterName,
-        namespaceBO.getBaseInfo().getNamespaceName(),
-        NamespaceBOUtils.convert2configFileContent(namespaceBO),
-        ConfigFileFormat.fromString(namespaceBO.getFormat())
+         namespaceBO.getBaseInfo().getNamespaceName(), namespaceBO.isPublic(),
+         NamespaceBOUtils.convert2configFileContent(namespaceBO),
+         ConfigFileFormat.fromString(namespaceBO.getFormat())
     );
   }
 
@@ -67,6 +70,7 @@ public String toString() {
         ", appId='" + appId + '\'' +
         ", clusterName='" + clusterName + '\'' +
         ", namespace='" + namespace + '\'' +
+        ", isPublic='" + isPublic + '\'' +
         ", configFileContent='" + configFileContent + '\'' +
         ", format=" + format +
         '}';
@@ -99,4 +103,8 @@ public String getConfigFileContent() {
   public ConfigFileFormat getFormat() {
     return format;
   }
+
+  public boolean isPublic() {
+    return isPublic;
+  }
 }
diff --git a/...rtal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java b/...rtal/src/main/java/com/ctrip/framework/apollo/portal/entity/model/NamespaceTextModel.java
@@ -30,6 +30,7 @@ public class NamespaceTextModel implements Verifiable {
   private long namespaceId;
   private String format;
   private String configText;
+  private String operator;
 
 
   @Override
@@ -92,4 +93,12 @@ public ConfigFileFormat getFormat() {
   public void setFormat(String format) {
     this.format = format;
   }
+
+  public String getOperator() {
+    return operator;
+  }
+
+  public void setOperator(String operator) {
+    this.operator = operator;
+  }
 }