Skip to content

Commit

Permalink
solve the problem of duplicate comments and blank lines (#5232)
Browse files Browse the repository at this point in the history 
* feat:
1. solve the problem of duplicate comments and blank lines
2. reduce the data growth of the item table
3. in the same case, the size of ItemChangeSets can be reduced
4. when revoke configuration, if there is only one comment or blank line, it will not be deleted.

* fix：
1. add CHANGES.md
2. PropertyResolver code style format
3. Revoke configurations cause the line num to be disordered

* fix：code optimization

* feat:
1. solve the problem of duplicate comments and blank lines
2. reduce the data growth of the item table
3. in the same case, the size of ItemChangeSets can be reduced
4. when revoke configuration, if there is only one comment or blank line, it will not be deleted.

* fix：
1. add CHANGES.md
2. PropertyResolver code style format
3. Revoke configurations cause the line num to be disordered

* fix：code optimization
  • Loading branch information
@youngzil
youngzil authored Oct 1, 2024
1 parent c6b165d commit a1c7c96
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 57 deletions.
3 changes: 2 additions & 1 deletion CHANGES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ Apollo 2.4.0
* [Update the server config link in system info page](https://github.com/apolloconfig/apollo/pull/5204)
* [Feature support portal restTemplate Client connection pool config](https://github.com/apolloconfig/apollo/pull/5200)
* [Feature added the ability for administrators to globally search for Value](https://github.com/apolloconfig/apollo/pull/5182)
* [Fix: Resolve issues with duplicate comments and blank lines in configuration management](https://github.com/apolloconfig/apollo/pull/5232)
* [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240)

------------------
All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
78 changes: 45 additions & 33 deletions ...c/main/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,20 @@
import com.ctrip.framework.apollo.common.exception.BadRequestException;
import com.ctrip.framework.apollo.common.utils.BeanUtils;

import com.ctrip.framework.apollo.core.utils.StringUtils;
import com.google.common.base.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import javax.validation.constraints.NotNull;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
* normal property file resolver.
Expand All @@ -45,12 +50,21 @@ public class PropertyResolver implements ConfigTextResolver {
@Override
public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO> baseItems) {

Map<Integer, ItemDTO> oldLineNumMapItem = BeanUtils.mapByKey("lineNum", baseItems);
Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);

//remove comment and blank item map.
oldKeyMapItem.remove("");

// comment items
List<ItemDTO> baseCommentItems = new LinkedList<>();
// blank items
List<ItemDTO> baseBlankItems = new LinkedList<>();
if (!CollectionUtils.isEmpty(baseItems)) {

baseCommentItems = baseItems.stream().filter(itemDTO -> isCommentItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toCollection(LinkedList::new));

baseBlankItems = baseItems.stream().filter(itemDTO -> isBlankItem(itemDTO)).sorted(Comparator.comparing(ItemDTO::getLineNum)).collect(Collectors.toCollection(LinkedList::new));
}

String[] newItems = configText.split(ITEM_SEPARATOR);
Set<String> repeatKeys = new HashSet<>();
if (isHasRepeatKey(newItems, repeatKeys)) {
Expand All @@ -63,17 +77,25 @@ public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO>
for (String newItem : newItems) {
newItem = newItem.trim();
newLineNumMapItem.put(lineCounter, newItem);
ItemDTO oldItemByLine = oldLineNumMapItem.get(lineCounter);

//comment item
if (isCommentItem(newItem)) {
ItemDTO oldItemDTO = null;
if (!CollectionUtils.isEmpty(baseCommentItems)) {
oldItemDTO = baseCommentItems.remove(0);
}

handleCommentLine(namespaceId, oldItemByLine, newItem, lineCounter, changeSets);
handleCommentLine(namespaceId, oldItemDTO, newItem, lineCounter, changeSets);

//blank item
} else if (isBlankItem(newItem)) {

handleBlankLine(namespaceId, oldItemByLine, lineCounter, changeSets);
ItemDTO oldItemDTO = null;
if (!CollectionUtils.isEmpty(baseBlankItems)) {
oldItemDTO = baseBlankItems.remove(0);
}

handleBlankLine(namespaceId, oldItemDTO, lineCounter, changeSets);

//normal item
} else {
Expand All @@ -83,7 +105,7 @@ public ItemChangeSets resolve(long namespaceId, String configText, List<ItemDTO>
lineCounter++;
}

deleteCommentAndBlankItem(oldLineNumMapItem, newLineNumMapItem, changeSets);
deleteCommentAndBlankItem(baseCommentItems, baseBlankItems, changeSets);
deleteNormalKVItem(oldKeyMapItem, changeSets);

return changeSets;
Expand All @@ -97,7 +119,7 @@ private boolean isHasRepeatKey(String[] newItems, @NotNull Set<String> repeatKey
String[] kv = parseKeyValueFromItem(item);
if (kv != null) {
String key = kv[0].toLowerCase();
if(!keys.add(key)){
if (!keys.add(key)) {
repeatKeys.add(key);
}
} else {
Expand All @@ -122,16 +144,18 @@ private String[] parseKeyValueFromItem(String item) {
}

private void handleCommentLine(Long namespaceId, ItemDTO oldItemByLine, String newItem, int lineCounter, ItemChangeSets changeSets) {
String oldComment = oldItemByLine == null ? "" : oldItemByLine.getComment();
//create comment. implement update comment by delete old comment and create new comment
if (!(isCommentItem(oldItemByLine) && newItem.equals(oldComment))) {
if (null == oldItemByLine) {
changeSets.addCreateItem(buildCommentItem(0L, namespaceId, newItem, lineCounter));
} else if (!StringUtils.equals(oldItemByLine.getComment(), newItem) || lineCounter != oldItemByLine.getLineNum()) {
changeSets.addUpdateItem(buildCommentItem(oldItemByLine.getId(), namespaceId, newItem, lineCounter));
}
}

private void handleBlankLine(Long namespaceId, ItemDTO oldItem, int lineCounter, ItemChangeSets changeSets) {
if (!isBlankItem(oldItem)) {
if (null == oldItem) {
changeSets.addCreateItem(buildBlankItem(0L, namespaceId, lineCounter));
} else if (lineCounter != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildBlankItem(oldItem.getId(), namespaceId, lineCounter));
}
}

Expand All @@ -149,12 +173,12 @@ private void handleNormalLine(Long namespaceId, Map<String, ItemDTO> keyMapOldIt

ItemDTO oldItem = keyMapOldItem.get(newKey);

if (oldItem == null) {//new item
//new item
if (oldItem == null) {
changeSets.addCreateItem(buildNormalItem(0L, namespaceId, newKey, newValue, "", lineCounter));
} else if (!newValue.equals(oldItem.getValue()) || lineCounter != oldItem.getLineNum()) {//update item
changeSets.addUpdateItem(
buildNormalItem(oldItem.getId(), namespaceId, newKey, newValue, oldItem.getComment(),
lineCounter));
//update item
} else if (!StringUtils.equals(newValue, oldItem.getValue()) || lineCounter != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildNormalItem(oldItem.getId(), namespaceId, newKey, newValue, oldItem.getComment(), lineCounter));
}
keyMapOldItem.remove(newKey);
}
Expand All @@ -173,7 +197,7 @@ private boolean isBlankItem(ItemDTO item) {
}

private boolean isBlankItem(String line) {
return Strings.nullToEmpty(line).trim().isEmpty();
return Strings.nullToEmpty(line).trim().isEmpty();
}

private void deleteNormalKVItem(Map<String, ItemDTO> baseKeyMapItem, ItemChangeSets changeSets) {
Expand All @@ -183,23 +207,11 @@ private void deleteNormalKVItem(Map<String, ItemDTO> baseKeyMapItem, ItemChangeS
}
}

private void deleteCommentAndBlankItem(Map<Integer, ItemDTO> oldLineNumMapItem,
Map<Integer, String> newLineNumMapItem,
private void deleteCommentAndBlankItem(List<ItemDTO> baseCommentItems,
List<ItemDTO> baseBlankItems,
ItemChangeSets changeSets) {

for (Map.Entry<Integer, ItemDTO> entry : oldLineNumMapItem.entrySet()) {
int lineNum = entry.getKey();
ItemDTO oldItem = entry.getValue();
String newItem = newLineNumMapItem.get(lineNum);

//1. old is blank by now is not
//2.old is comment by now is not exist or modified
//3.old is blank by now is not exist or modified
if ((isBlankItem(oldItem) && !isBlankItem(newItem))
|| (isCommentItem(oldItem) || isBlankItem(oldItem)) && (newItem == null || !newItem.equals(oldItem.getComment()))) {
changeSets.addDeleteItem(oldItem);
}
}
baseCommentItems.forEach(oldItemDTO -> changeSets.addDeleteItem(oldItemDTO));
baseBlankItems.forEach(oldItemDTO -> changeSets.addDeleteItem(oldItemDTO));
}

private ItemDTO buildCommentItem(Long id, Long namespaceId, String comment, int lineNum) {
Expand Down
21 changes: 11 additions & 10 deletions apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ItemService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@Service
public class ItemService {
Expand Down Expand Up @@ -216,29 +217,29 @@ public void revokeItem(String appId, Env env, String clusterName, String namespa
}
List<ItemDTO> baseItems = itemAPI.findItems(appId, env, clusterName, namespaceName);
Map<String, ItemDTO> oldKeyMapItem = BeanUtils.mapByKey("key", baseItems);
Map<String, ItemDTO> deletedItemDTOs = new HashMap<>();
//remove comment and blank item map.
oldKeyMapItem.remove("");

//deleted items for comment
findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> {
deletedItemDTOs.put(item.getKey(),item);
});
Map<String, ItemDTO> deletedItemDTOs = findDeletedItems(appId, env, clusterName, namespaceName).stream()
.filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey()))
.collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2));

ItemChangeSets changeSets = new ItemChangeSets();
AtomicInteger lineNum = new AtomicInteger(1);
releaseItemDTOs.forEach((key,value) -> {
ItemDTO oldItem = oldKeyMapItem.get(key);
if (oldItem == null) {
ItemDTO deletedItemDto = deletedItemDTOs.computeIfAbsent(key, k -> new ItemDTO());
changeSets.addCreateItem(buildNormalItem(0L, namespaceId,key,value,deletedItemDto.getComment(),lineNum.get()));
} else if (!oldItem.getValue().equals(value) || lineNum.get() != oldItem
.getLineNum()) {
changeSets.addUpdateItem(buildNormalItem(oldItem.getId(), namespaceId, key,
value, oldItem.getComment(), lineNum.get()));
int newLineNum = 0 == deletedItemDto.getLineNum() ? lineNum.get() : deletedItemDto.getLineNum();
changeSets.addCreateItem(buildNormalItem(0L, namespaceId, key, value, deletedItemDto.getComment(), newLineNum));
} else if (!StringUtils.equals(oldItem.getValue(), value) || lineNum.get() != oldItem.getLineNum()) {
changeSets.addUpdateItem(buildNormalItem(oldItem.getId(), namespaceId, key, value, oldItem.getComment(), oldItem.getLineNum()));
}
oldKeyMapItem.remove(key);
lineNum.set(lineNum.get() + 1);
});
oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(oldKeyMapItem.get(key)));
oldKeyMapItem.forEach((key, value) -> changeSets.addDeleteItem(value));
changeSets.setDataChangeLastModifiedBy(userInfoHolder.getUser().getUserId());

updateItems(appId, env, clusterName, namespaceName, changeSets);
Expand Down
9 changes: 5 additions & 4 deletions apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/NamespaceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,6 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole
//latest Release
ReleaseDTO latestRelease;
Map<String, String> releaseItems = new HashMap<>();
Map<String, ItemDTO> deletedItemDTOs = new HashMap<>();
latestRelease = releaseService.loadLatestRelease(appId, env, clusterName, namespaceName);
if (latestRelease != null) {
releaseItems = GSON.fromJson(latestRelease.getConfigurations(), GsonType.CONFIG);
Expand All @@ -333,9 +332,9 @@ private NamespaceBO transformNamespace2BO(Env env, NamespaceDTO namespace, boole

if (includeDeletedItems) {
//deleted items
itemService.findDeletedItems(appId, env, clusterName, namespaceName).forEach(item -> {
deletedItemDTOs.put(item.getKey(), item);
});
Map<String, ItemDTO> deletedItemDTOs = itemService.findDeletedItems(appId, env, clusterName, namespaceName).stream()
.filter(itemDTO -> !StringUtils.isEmpty(itemDTO.getKey()))
.collect(Collectors.toMap(itemDTO -> itemDTO.getKey(), v -> v, (v1, v2) -> v2));

List<ItemBO> deletedItems = parseDeletedItems(items, releaseItems, deletedItemDTOs);
itemBOs.addAll(deletedItems);
Expand Down Expand Up @@ -385,6 +384,8 @@ private void fillAppNamespaceProperties(NamespaceBO namespace) {

private List<ItemBO> parseDeletedItems(List<ItemDTO> newItems, Map<String, String> releaseItems, Map<String, ItemDTO> deletedItemDTOs) {
Map<String, ItemDTO> newItemMap = BeanUtils.mapByKey("key", newItems);
//remove comment and blank item map.
newItemMap.remove("");

List<ItemBO> deletedItems = new LinkedList<>();
for (Map.Entry<String, String> entry : releaseItems.entrySet()) {
Expand Down
18 changes: 9 additions & 9 deletions ...st/java/com/ctrip/framework/apollo/portal/component/txtresolver/PropertyResolverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ public void testDeleteItem() {
@Test
public void testDeleteCommentItem() {
ItemChangeSets changeSets = resolver.resolve(1, "a=b\n\nb=c", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(2, changeSets.getDeleteItems().size());
Assert.assertEquals(2, changeSets.getUpdateItems().size());
Assert.assertEquals(1, changeSets.getCreateItems().size());
Assert.assertEquals(1, changeSets.getDeleteItems().size());
Assert.assertEquals(3, changeSets.getUpdateItems().size());
Assert.assertEquals(0, changeSets.getCreateItems().size());
}

@Test
Expand All @@ -120,17 +120,17 @@ public void testUpdateCommentItem() {
+ "a=b\n"
+"\n"
+ "b=c", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(1, changeSets.getDeleteItems().size());
Assert.assertEquals(0, changeSets.getUpdateItems().size());
Assert.assertEquals(1, changeSets.getCreateItems().size());
Assert.assertEquals(0, changeSets.getDeleteItems().size());
Assert.assertEquals(1, changeSets.getUpdateItems().size());
Assert.assertEquals(0, changeSets.getCreateItems().size());
}

@Test
public void testAllSituation(){
ItemChangeSets changeSets = resolver.resolve(1, "#ww\nd=e\nb=c\na=b\n\nq=w\n#eee", mockBaseItemWith2Key1Comment1Blank());
Assert.assertEquals(2, changeSets.getDeleteItems().size());
Assert.assertEquals(2, changeSets.getUpdateItems().size());
Assert.assertEquals(5, changeSets.getCreateItems().size());
Assert.assertEquals(0, changeSets.getDeleteItems().size());
Assert.assertEquals(4, changeSets.getUpdateItems().size());
Assert.assertEquals(3, changeSets.getCreateItems().size());
}

/**
Expand Down

0 comments on commit a1c7c96

Please sign in to comment.