diff --git a/CHANGES.md b/CHANGES.md index 20d88a3a536..4d97eead265 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,7 @@ Apollo 2.2.0 * [Misc dependency updates](https://github.com/apolloconfig/apollo/pull/4784) * [Fix the problem that the deletion failure of the system rights management page does not prompt](https://github.com/apolloconfig/apollo/pull/4803) * [Fix the issue of the system permission management page retrieving non-existent users](https://github.com/apolloconfig/apollo/pull/4802) +* [Add release history cleaning function](https://github.com/apolloconfig/apollo/pull/4813) * [[Multi-Database Support][pg] Make JdbcUserDetailsManager compat with postgre](https://github.com/apolloconfig/apollo/pull/4790) ------------------ diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index e9a12e9d0d7..1d87cd07e07 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.springframework.stereotype.Component; @Component @@ -46,12 +47,16 @@ public class BizConfig extends RefreshableConfig { private static final int DEFAULT_RELEASE_MESSAGE_NOTIFICATION_BATCH = 100; private static final int DEFAULT_RELEASE_MESSAGE_NOTIFICATION_BATCH_INTERVAL_IN_MILLI = 100;//100ms private static final int DEFAULT_LONG_POLLING_TIMEOUT = 60; //60s + public static final int DEFAULT_RELEASE_HISTORY_RETENTION_SIZE = -1; private static final Gson GSON = new Gson(); private static final Type namespaceValueLengthOverrideTypeReference = new TypeToken>() { }.getType(); + private static final Type releaseHistoryRetentionSizeOverrideTypeReference = + new TypeToken>() { + }.getType(); private final BizDBPropertySource propertySource; @@ -154,6 +159,24 @@ public int accessKeyAuthTimeDiffTolerance() { DEFAULT_ACCESS_KEY_AUTH_TIME_DIFF_TOLERANCE); } + public int releaseHistoryRetentionSize() { + int count = getIntProperty("apollo.release-history.retention.size", DEFAULT_RELEASE_HISTORY_RETENTION_SIZE); + return checkInt(count, 1, Integer.MAX_VALUE, DEFAULT_RELEASE_HISTORY_RETENTION_SIZE); + } + + public Map releaseHistoryRetentionSizeOverride() { + String overrideString = getValue("apollo.release-history.retention.size.override"); + Map releaseHistoryRetentionSizeOverride = Maps.newHashMap(); + if (!Strings.isNullOrEmpty(overrideString)) { + releaseHistoryRetentionSizeOverride = + GSON.fromJson(overrideString, releaseHistoryRetentionSizeOverrideTypeReference); + } + return releaseHistoryRetentionSizeOverride.entrySet() + .stream() + .filter(entry -> entry.getValue() >= 1) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + public int releaseMessageCacheScanInterval() { int interval = getIntProperty("apollo.release-message-cache-scan.interval", DEFAULT_RELEASE_MESSAGE_CACHE_SCAN_INTERVAL); return checkInt(interval, 1, Integer.MAX_VALUE, DEFAULT_RELEASE_MESSAGE_CACHE_SCAN_INTERVAL); diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepository.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepository.java index bd822d32252..626e51d892a 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepository.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepository.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.biz.repository; import com.ctrip.framework.apollo.biz.entity.ReleaseHistory; - +import java.util.List; import java.util.Set; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -42,4 +42,8 @@ Page findByAppIdAndClusterNameAndNamespaceNameOrderByIdDesc(Stri @Query("update ReleaseHistory set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?4 where AppId=?1 and ClusterName=?2 and NamespaceName = ?3 and IsDeleted = false") int batchDelete(String appId, String clusterName, String namespaceName, String operator); + Page findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(String appId, String clusterName, String namespaceName, String branchName, Pageable pageable); + + List findFirst100ByAppIdAndClusterNameAndNamespaceNameAndBranchNameAndIdLessThanEqualOrderByIdAsc(String appId, String clusterName, String namespaceName, String branchName, long maxId); + } diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryService.java index 80f200db2bf..9bc55577bab 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryService.java @@ -16,36 +16,92 @@ */ package com.ctrip.framework.apollo.biz.service; +import static com.ctrip.framework.apollo.biz.config.BizConfig.DEFAULT_RELEASE_HISTORY_RETENTION_SIZE; + +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Audit; import com.ctrip.framework.apollo.biz.entity.ReleaseHistory; import com.ctrip.framework.apollo.biz.repository.ReleaseHistoryRepository; +import com.ctrip.framework.apollo.biz.repository.ReleaseRepository; +import com.ctrip.framework.apollo.core.utils.ApolloThreadFactory; +import com.ctrip.framework.apollo.tracer.Tracer; +import com.google.common.collect.Queues; import com.google.gson.Gson; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; +import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import java.util.Date; import java.util.Map; import java.util.Set; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; +import org.springframework.transaction.support.TransactionTemplate; /** * @author Jason Song(song_s@ctrip.com) */ @Service public class ReleaseHistoryService { + private static final Logger logger = LoggerFactory.getLogger(ReleaseHistoryService.class); private static final Gson GSON = new Gson(); + private static final int CLEAN_QUEUE_MAX_SIZE = 100; + private final BlockingQueue releaseClearQueue = Queues.newLinkedBlockingQueue(CLEAN_QUEUE_MAX_SIZE); + private final ExecutorService cleanExecutorService = Executors.newSingleThreadExecutor( + ApolloThreadFactory.create("ReleaseHistoryService", true)); + private final AtomicBoolean cleanStopped = new AtomicBoolean(false); private final ReleaseHistoryRepository releaseHistoryRepository; + private final ReleaseRepository releaseRepository; private final AuditService auditService; + private final BizConfig bizConfig; + private final TransactionTemplate transactionManager; public ReleaseHistoryService( final ReleaseHistoryRepository releaseHistoryRepository, - final AuditService auditService) { + final ReleaseRepository releaseRepository, + final AuditService auditService, + final BizConfig bizConfig, + final TransactionTemplate transactionManager) { this.releaseHistoryRepository = releaseHistoryRepository; + this.releaseRepository = releaseRepository; this.auditService = auditService; + this.bizConfig = bizConfig; + this.transactionManager = transactionManager; } + @PostConstruct + private void initialize() { + cleanExecutorService.submit(() -> { + while (!cleanStopped.get() && !Thread.currentThread().isInterrupted()) { + try { + ReleaseHistory releaseHistory = releaseClearQueue.poll(1, TimeUnit.SECONDS); + if (releaseHistory != null) { + this.cleanReleaseHistory(releaseHistory); + } else { + TimeUnit.MINUTES.sleep(1); + } + } catch (Throwable ex) { + logger.error("Clean releaseHistory failed", ex); + Tracer.logError(ex); + } + } + }); + } public Page findReleaseHistoriesByNamespace(String appId, String clusterName, String namespaceName, Pageable @@ -92,6 +148,13 @@ public ReleaseHistory createReleaseHistory(String appId, String clusterName, Str auditService.audit(ReleaseHistory.class.getSimpleName(), releaseHistory.getId(), Audit.OP.INSERT, releaseHistory.getDataChangeCreatedBy()); + int releaseHistoryRetentionLimit = this.getReleaseHistoryRetentionLimit(releaseHistory); + if (releaseHistoryRetentionLimit != DEFAULT_RELEASE_HISTORY_RETENTION_SIZE) { + if (!releaseClearQueue.offer(releaseHistory)) { + logger.warn("releaseClearQueue is full, failed to add task to clean queue, " + + "clean queue max size:{}", CLEAN_QUEUE_MAX_SIZE); + } + } return releaseHistory; } @@ -99,4 +162,72 @@ public ReleaseHistory createReleaseHistory(String appId, String clusterName, Str public int batchDelete(String appId, String clusterName, String namespaceName, String operator) { return releaseHistoryRepository.batchDelete(appId, clusterName, namespaceName, operator); } + + private Optional releaseHistoryRetentionMaxId(ReleaseHistory releaseHistory, int releaseHistoryRetentionSize) { + Page releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc( + releaseHistory.getAppId(), + releaseHistory.getClusterName(), + releaseHistory.getNamespaceName(), + releaseHistory.getBranchName(), + PageRequest.of(releaseHistoryRetentionSize, 1) + ); + if (releaseHistoryPage.isEmpty()) { + return Optional.empty(); + } + return Optional.of( + releaseHistoryPage + .getContent() + .get(0) + .getId() + ); + } + + private void cleanReleaseHistory(ReleaseHistory cleanRelease) { + String appId = cleanRelease.getAppId(); + String clusterName = cleanRelease.getClusterName(); + String namespaceName = cleanRelease.getNamespaceName(); + String branchName = cleanRelease.getBranchName(); + + int retentionLimit = this.getReleaseHistoryRetentionLimit(cleanRelease); + //Second check, if retentionLimit is default value, do not clean + if (retentionLimit == DEFAULT_RELEASE_HISTORY_RETENTION_SIZE) { + return; + } + + Optional maxId = this.releaseHistoryRetentionMaxId(cleanRelease, retentionLimit); + if (!maxId.isPresent()) { + return; + } + + boolean hasMore = true; + while (hasMore && !Thread.currentThread().isInterrupted()) { + List cleanReleaseHistoryList = releaseHistoryRepository.findFirst100ByAppIdAndClusterNameAndNamespaceNameAndBranchNameAndIdLessThanEqualOrderByIdAsc( + appId, clusterName, namespaceName, branchName, maxId.get()); + Set releaseIds = cleanReleaseHistoryList.stream() + .map(ReleaseHistory::getReleaseId) + .collect(Collectors.toSet()); + + transactionManager.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(TransactionStatus status) { + releaseHistoryRepository.deleteAll(cleanReleaseHistoryList); + releaseRepository.deleteAllById(releaseIds); + } + }); + hasMore = cleanReleaseHistoryList.size() == 100; + } + } + + private int getReleaseHistoryRetentionLimit(ReleaseHistory releaseHistory) { + String overrideKey = String.format("%s+%s+%s+%s", releaseHistory.getAppId(), + releaseHistory.getClusterName(), releaseHistory.getNamespaceName(), releaseHistory.getBranchName()); + + Map overrideMap = bizConfig.releaseHistoryRetentionSizeOverride(); + return overrideMap.getOrDefault(overrideKey, bizConfig.releaseHistoryRetentionSize()); + } + + @PreDestroy + void stopClean() { + cleanStopped.set(true); + } } diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/AbstractIntegrationTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/AbstractIntegrationTest.java index 6ed07353eb6..612a6ea8edd 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/AbstractIntegrationTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/AbstractIntegrationTest.java @@ -32,4 +32,9 @@ ) public abstract class AbstractIntegrationTest { + protected static final String APP_ID = "kl-app"; + protected static final String CLUSTER_NAME = "default"; + protected static final String NAMESPACE_NAME = "application"; + protected static final String BRANCH_NAME = "default"; + } diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java index 840df6e2923..807e472a964 100644 --- a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/config/BizConfigTest.java @@ -69,6 +69,33 @@ public void testReleaseMessageNotificationBatchWithInvalidNumber() throws Except assertEquals(defaultBatch, bizConfig.releaseMessageNotificationBatch()); } + @Test + public void testReleaseHistoryRetentionSize() { + int someLimit = 20; + when(environment.getProperty("apollo.release-history.retention.size")).thenReturn(String.valueOf(someLimit)); + + assertEquals(someLimit, bizConfig.releaseHistoryRetentionSize()); + } + + @Test + public void testReleaseHistoryRetentionSizeOverride() { + int someOverrideLimit = 10; + String overrideValueString = "{'a+b+c+b':10}"; + when(environment.getProperty("apollo.release-history.retention.size.override")).thenReturn(overrideValueString); + int overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+c+b"); + assertEquals(someOverrideLimit, overrideValue); + + overrideValueString = "{'a+b+c+b':0,'a+b+d+b':2}"; + when(environment.getProperty("apollo.release-history.retention.size.override")).thenReturn(overrideValueString); + assertEquals(1, bizConfig.releaseHistoryRetentionSizeOverride().size()); + overrideValue = bizConfig.releaseHistoryRetentionSizeOverride().get("a+b+d+b"); + assertEquals(2, overrideValue); + + overrideValueString = "{}"; + when(environment.getProperty("apollo.release-history.retention.size.override")).thenReturn(overrideValueString); + assertEquals(0, bizConfig.releaseHistoryRetentionSizeOverride().size()); + } + @Test public void testReleaseMessageNotificationBatchWithNAN() throws Exception { String someNAN = "someNAN"; diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepositoryTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepositoryTest.java new file mode 100644 index 00000000000..58f95f80bda --- /dev/null +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/repository/ReleaseHistoryRepositoryTest.java @@ -0,0 +1,78 @@ +/* + * Copyright 2023 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.biz.repository; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.ctrip.framework.apollo.biz.AbstractIntegrationTest; +import com.ctrip.framework.apollo.biz.entity.ReleaseHistory; +import java.util.List; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.Sql.ExecutionPhase; + +/** + * @author kl (http://kailing.pub) + * @since 2023/3/23 + */ +public class ReleaseHistoryRepositoryTest extends AbstractIntegrationTest { + + @Autowired + private ReleaseHistoryRepository releaseHistoryRepository; + + @Test + @Sql(scripts = "/sql/release-history-test.sql", executionPhase = ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = ExecutionPhase.AFTER_TEST_METHOD) + public void testFindReleaseHistoryRetentionMaxId() { + Page releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, PageRequest.of(1, 1)); + assertEquals(5, releaseHistoryPage.getContent().get(0).getId()); + + releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, PageRequest.of(2, 1)); + assertEquals(4, releaseHistoryPage.getContent().get(0).getId()); + + releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, PageRequest.of(5, 1)); + assertEquals(1, releaseHistoryPage.getContent().get(0).getId()); + + releaseHistoryRepository.deleteAll(); + releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, PageRequest.of(1, 1)); + assertTrue(releaseHistoryPage.isEmpty()); + } + + @Test + @Sql(scripts = "/sql/release-history-test.sql", executionPhase = ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = ExecutionPhase.AFTER_TEST_METHOD) + public void testFindFirst100ByAppIdAndClusterNameAndNamespaceNameAndBranchNameAndIdLessThanEqualOrderByIdAsc() { + + int releaseHistoryRetentionSize = 2; + Page releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, PageRequest.of(releaseHistoryRetentionSize, 1)); + long releaseMaxId = releaseHistoryPage.getContent().get(0).getId(); + List releaseHistories = releaseHistoryRepository.findFirst100ByAppIdAndClusterNameAndNamespaceNameAndBranchNameAndIdLessThanEqualOrderByIdAsc( + APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, releaseMaxId); + assertEquals(4, releaseHistories.size()); + + releaseHistoryRetentionSize = 1; + releaseHistoryPage = releaseHistoryRepository.findByAppIdAndClusterNameAndNamespaceNameAndBranchNameOrderByIdDesc(APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, PageRequest.of(releaseHistoryRetentionSize, 1)); + releaseMaxId = releaseHistoryPage.getContent().get(0).getId(); + releaseHistories = releaseHistoryRepository.findFirst100ByAppIdAndClusterNameAndNamespaceNameAndBranchNameAndIdLessThanEqualOrderByIdAsc( + APP_ID, CLUSTER_NAME, NAMESPACE_NAME, BRANCH_NAME, releaseMaxId); + assertEquals(5, releaseHistories.size()); + } +} diff --git a/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryServiceTest.java b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryServiceTest.java new file mode 100644 index 00000000000..e66da46c3cc --- /dev/null +++ b/apollo-biz/src/test/java/com/ctrip/framework/apollo/biz/service/ReleaseHistoryServiceTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2023 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.biz.service; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.ctrip.framework.apollo.biz.BizTestConfiguration; +import com.ctrip.framework.apollo.biz.config.BizConfig; +import com.ctrip.framework.apollo.biz.entity.Release; +import com.ctrip.framework.apollo.biz.entity.ReleaseHistory; +import com.ctrip.framework.apollo.biz.repository.ReleaseHistoryRepository; +import com.ctrip.framework.apollo.biz.repository.ReleaseRepository; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.lang.reflect.Method; +import java.sql.SQLException; +import org.hibernate.exception.JDBCConnectionException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.springframework.aop.framework.AopProxyUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.annotation.DirtiesContext.ClassMode; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.Sql.ExecutionPhase; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.util.ReflectionUtils; + +/** + * @author kl (http://kailing.pub) + * @since 2023/3/24 + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest( + classes = BizTestConfiguration.class, + webEnvironment = WebEnvironment.RANDOM_PORT +) +@DirtiesContext(classMode = ClassMode.AFTER_CLASS) +public class ReleaseHistoryServiceTest { + + @Mock + private BizConfig bizConfig; + @Mock + private ReleaseRepository mockReleaseRepository; + + private ReleaseHistory mockReleaseHistory; + private static final String APP_ID = "kl-app"; + private static final String CLUSTER_NAME = "default"; + private static final String NAMESPACE_NAME = "application"; + private static final String BRANCH_NAME = "default"; + + @Autowired + private ReleaseHistoryService releaseHistoryService; + @Autowired + private ReleaseHistoryRepository releaseHistoryRepository; + @Autowired + private ReleaseRepository releaseRepository; + + @Before + public void setUp() throws Exception { + ReflectionTestUtils.setField(releaseHistoryService, "bizConfig", bizConfig); + mockReleaseHistory = spy(ReleaseHistory.class); + mockReleaseHistory.setBranchName(BRANCH_NAME); + mockReleaseHistory.setNamespaceName(NAMESPACE_NAME); + mockReleaseHistory.setClusterName(CLUSTER_NAME); + mockReleaseHistory.setAppId(APP_ID); + } + + @Test + @Sql(scripts = "/sql/release-history-test.sql", executionPhase = ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = ExecutionPhase.AFTER_TEST_METHOD) + public void testCleanReleaseHistory() { + ReleaseHistoryService service = (ReleaseHistoryService) AopProxyUtils.getSingletonTarget(releaseHistoryService); + assert service != null; + Method method = ReflectionUtils.findMethod(service.getClass(), "cleanReleaseHistory", ReleaseHistory.class); + assert method != null; + ReflectionUtils.makeAccessible(method); + + when(bizConfig.releaseHistoryRetentionSize()).thenReturn(-1); + when(bizConfig.releaseHistoryRetentionSizeOverride()).thenReturn(Maps.newHashMap()); + ReflectionUtils.invokeMethod(method, service, mockReleaseHistory); + Assert.assertEquals(6, releaseHistoryRepository.count()); + Assert.assertEquals(6, releaseRepository.count()); + + when(bizConfig.releaseHistoryRetentionSize()).thenReturn(2); + when(bizConfig.releaseHistoryRetentionSizeOverride()).thenReturn(Maps.newHashMap()); + ReflectionUtils.invokeMethod(method, service, mockReleaseHistory); + Assert.assertEquals(2, releaseHistoryRepository.count()); + Assert.assertEquals(2, releaseRepository.count()); + + when(bizConfig.releaseHistoryRetentionSize()).thenReturn(2); + when(bizConfig.releaseHistoryRetentionSizeOverride()).thenReturn( + ImmutableMap.of("kl-app+default+application+default", 1)); + ReflectionUtils.invokeMethod(method, service, mockReleaseHistory); + Assert.assertEquals(1, releaseHistoryRepository.count()); + Assert.assertEquals(1, releaseRepository.count()); + + Iterable historyList = releaseHistoryRepository.findAll(); + historyList.forEach(history -> Assert.assertEquals(6, history.getId())); + + Iterable releaseList = releaseRepository.findAll(); + releaseList.forEach(release -> Assert.assertEquals(6, release.getId())); + } + + @Test + @Sql(scripts = "/sql/release-history-test.sql", executionPhase = ExecutionPhase.BEFORE_TEST_METHOD) + @Sql(scripts = "/sql/clean.sql", executionPhase = ExecutionPhase.AFTER_TEST_METHOD) + public void testCleanReleaseHistoryTransactionalRollBack() { + ReleaseHistoryService service = (ReleaseHistoryService) AopProxyUtils.getSingletonTarget(releaseHistoryService); + assert service != null; + Method method = ReflectionUtils.findMethod(service.getClass(), "cleanReleaseHistory", ReleaseHistory.class); + assert method != null; + ReflectionUtils.makeAccessible(method); + + when(bizConfig.releaseHistoryRetentionSize()).thenReturn(1); + when(bizConfig.releaseHistoryRetentionSizeOverride()).thenReturn(Maps.newHashMap()); + ReflectionTestUtils.setField(releaseHistoryService, "releaseRepository", mockReleaseRepository); + doThrow(new JDBCConnectionException("error", new SQLException("sql"))).when(mockReleaseRepository).deleteAllById(any()); + Assert.assertThrows(JDBCConnectionException.class, () -> + ReflectionUtils.invokeMethod(method, service, mockReleaseHistory)); + + Assert.assertEquals(6, releaseHistoryRepository.count()); + + ReflectionTestUtils.setField(releaseHistoryService, "releaseRepository", releaseRepository); + Assert.assertEquals(6, releaseRepository.count()); + + ReflectionUtils.invokeMethod(method, service, mockReleaseHistory); + Assert.assertEquals(1, releaseHistoryRepository.count()); + Assert.assertEquals(1, releaseRepository.count()); + } + +} diff --git a/apollo-biz/src/test/resources/sql/release-history-test.sql b/apollo-biz/src/test/resources/sql/release-history-test.sql new file mode 100644 index 00000000000..411e05068cd --- /dev/null +++ b/apollo-biz/src/test/resources/sql/release-history-test.sql @@ -0,0 +1,33 @@ +-- +-- Copyright 2023 Apollo Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +INSERT INTO ReleaseHistory (Id, AppId, ClusterName, NamespaceName, BranchName, ReleaseId, + PreviousReleaseId, Operation, OperationContext, + DataChange_CreatedBy, DataChange_LastModifiedBy) +VALUES (1, 'kl-app', 'default', 'application', 'default', 1, 0, 0, '{"isEmergencyPublish":false}', 'apollo', 'apollo'), + (2, 'kl-app', 'default', 'application', 'default', 2, 1, 0, '{"isEmergencyPublish":false}', 'apollo', 'apollo'), + (3, 'kl-app', 'default', 'application', 'default', 3, 2, 0, '{"isEmergencyPublish":false}', 'apollo', 'apollo'), + (4, 'kl-app', 'default', 'application', 'default', 4, 3, 0, '{"isEmergencyPublish":false}', 'apollo', 'apollo'), + (5, 'kl-app', 'default', 'application', 'default', 5, 4, 0, '{"isEmergencyPublish":false}', 'apollo', 'apollo'), + (6, 'kl-app', 'default', 'application', 'default', 6, 5, 0, '{"isEmergencyPublish":false}', 'apollo', 'apollo'); + +INSERT INTO "Release" (Id, ReleaseKey, Name, Comment, AppId, ClusterName, NamespaceName, Configurations) +VALUES (1, 'TEST-RELEASE-KEY1', 'test','First Release','kl-app', 'default', 'application', '{"k1":"override-someDC-v1"}'), + (2, 'TEST-RELEASE-KEY2', 'test','First Release','kl-app', 'default', 'application', '{"k1":"override-someDC-v1"}'), + (3, 'TEST-RELEASE-KEY3', 'test','First Release','kl-app', 'default', 'application', '{"k1":"override-someDC-v1"}'), + (4, 'TEST-RELEASE-KEY4', 'test','First Release','kl-app', 'default', 'application', '{"k1":"override-someDC-v1"}'), + (5, 'TEST-RELEASE-KEY5', 'test','First Release','kl-app', 'default', 'application', '{"k1":"override-someDC-v1"}'), + (6, 'TEST-RELEASE-KEY6', 'test','First Release','kl-app', 'default', 'application', '{"k1":"override-someDC-v1"}'); + diff --git a/docs/en/deployment/distributed-deployment-guide.md b/docs/en/deployment/distributed-deployment-guide.md index 1bd77e973c8..59d011490c6 100644 --- a/docs/en/deployment/distributed-deployment-guide.md +++ b/docs/en/deployment/distributed-deployment-guide.md @@ -1580,3 +1580,23 @@ A reboot is required to take effect after the modification. Configure the login password of eureka server, which needs to be used together with [apollo.eureka.server.security.enabled](#_329-apolloeurekaserversecurityenabled-configure-whether-to-enable-eureka-login-authentication). A reboot is required to take effect after the modification. + +### 3.2.12 apollo.release-history.retention.size - Number of retained configurations release history + +> For version 2.2.0 and above + +The default value is -1, which means there is no limit on the number of retained release history. If the configuration is set to a positive integer(The minimum value is 1, which means at least one record of history must be kept to ensure the basic configuration functionality), only the specified number of recent release histories will be kept. This is to prevent excessive database pressure caused by too many release histories. It is recommended to configure this value based on the business needs for configuration rollback. This configuration item is global and cleaned up based on appId + clusterName + namespaceName + branchName. + +### 3.2.13 apollo.release-history.retention.size.override - Number of retained configurations release history at a granular level + +> For version 2.2.0 and above + +This configuration is used to override the `apollo.release-history.retention.size` configuration and achieve granular control over the number of retained release histories for appId+clusterName+namespaceName+branchName. The value of this configuration is in JSON format, with the JSON key being the concatenated value of appId, clusterName, namespaceName, and branchName using a `+` sign. The format is as follows: +``` +json +{ + "kl+bj+namespace1+bj": 10, + "kl+bj+namespace2+bj": 20 +} +``` +The above configuration specifies that the retention size for release history of appId=kl, clusterName=bj, namespaceName=namespace1, and branchName=bj is 10, and the retention size for release history of appId=kl, clusterName=bj, namespaceName=namespace2, and branchName=bj is 20. In general, branchName equals clusterName. It is only different during gray release, where the branchName needs to be confirmed by querying the ReleaseHistory table in the database. \ No newline at end of file diff --git a/docs/zh/deployment/distributed-deployment-guide.md b/docs/zh/deployment/distributed-deployment-guide.md index 0b653ad153d..5da79cd3272 100644 --- a/docs/zh/deployment/distributed-deployment-guide.md +++ b/docs/zh/deployment/distributed-deployment-guide.md @@ -1518,3 +1518,23 @@ http://some-user-name:some-password@1.1.1.1:8080/eureka/,http://some-user-name:s 配置eureka server的登录密码，需要和[apollo.eureka.server.security.enabled](#_329-apolloeurekaserversecurityenabled-配置是否开启eureka-server的登录认证)一起使用。 修改完需要重启生效。 + +### 3.2.12 apollo.release-history.retention.size - 配置发布历史的保留数量 + +> 适用于2.2.0及以上版本 + +默认为 -1，表示不限制保留数量。如果配置为正整数(最小值为 1，必须保留一条历史记录，保障基本的配置功能)，则只会保留最近的指定数量的发布历史。这是为了防止发布历史过多导致数据库压力过大，建议根据业务对配置回滚的需求来配置该值。该配置项是全局的，清理时是以 appId+clusterName+namespaceName+branchName 为维度清理的。 + +### 3.2.13 apollo.release-history.retention.size.override - 细粒度配置发布历史的保留数量 + +> 适用于2.2.0及以上版本 + +此配置用来覆盖 `apollo.release-history.retention.size` 的配置，做到细粒度控制 appId+clusterName+namespaceName+branchName 的发布历史保留数量，配置的值是一个 JSON 格式，JSON 的 key 为 appId、clusterName、namespaceName、branchName 使用 + 号的拼接值，格式如下： +``` +json +{ + "kl+bj+namespace1+bj": 10, + "kl+bj+namespace2+bj": 20 +} +``` +以上配置指定了 appId=kl、clusterName=bj、namespaceName=namespace1、branchName=bj 的发布历史保留数量为 10，appId=kl、clusterName=bj、namespaceName=namespace2、branchName=bj 的发布历史保留数量为 20，branchName 一般等于 clusterName，只有灰度发布时才会不同，灰度发布的 branchName 需要查询数据库 ReleaseHistory 表确认。 \ No newline at end of file