feat: support incremental configuration synchronization client

.vscode/launch.json

@@ -0,0 +1,77 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "java",
+            "name": "Current File",
+            "request": "launch",
+            "mainClass": "${file}"
+        },
+        {
+            "type": "java",
+            "name": "LocalAdminServiceApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.LocalAdminServiceApplication",
+            "projectName": "apollo-adminservice"
+        },
+        {
+            "type": "java",
+            "name": "AdminServiceApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.adminservice.AdminServiceApplication",
+            "projectName": "apollo-adminservice"
+        },
+        {
+            "type": "java",
+            "name": "ApolloApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.assembly.ApolloApplication",
+            "projectName": "apollo-assembly"
+        },
+        {
+            "type": "java",
+            "name": "LocalApolloApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.assembly.LocalApolloApplication",
+            "projectName": "apollo-assembly"
+        },
+        {
+            "type": "java",
+            "name": "ApolloSqlConverter",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.build.sql.converter.ApolloSqlConverter",
+            "projectName": "apollo-build-sql-converter"
+        },
+        {
+            "type": "java",
+            "name": "LocalConfigServiceApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.LocalConfigServiceApplication",
+            "projectName": "apollo-configservice"
+        },
+        {
+            "type": "java",
+            "name": "ConfigServiceApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.configservice.ConfigServiceApplication",
+            "projectName": "apollo-configservice"
+        },
+        {
+            "type": "java",
+            "name": "LocalPortalApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.LocalPortalApplication",
+            "projectName": "apollo-portal"
+        },
+        {
+            "type": "java",
+            "name": "PortalApplication",
+            "request": "launch",
+            "mainClass": "com.ctrip.framework.apollo.portal.PortalApplication",
+            "projectName": "apollo-portal"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "java.compile.nullAnalysis.mode": "automatic",
+    "java.configuration.updateBuildConfiguration": "automatic"
+}

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java

@@ -267,6 +267,10 @@ public long getInstanceConfigAuditTimeThresholdInMilli() {
     return TimeUnit.MINUTES.toMillis(timeThreshold);
   }
 
+  public boolean isConfigServiceIncrementalChangeEnabled() {
+    return getBooleanProperty("config-service.incremental.change.enabled", false);
+  }
+
   int checkInt(int value, int min, int max, int defaultValue) {
     if (value >= min && value <= max) {
       return value;

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseRepository.java

@@ -37,13 +37,15 @@ Release findFirstByAppIdAndClusterNameAndNamespaceNameAndIsAbandonedFalseOrderBy
 
   Release findByIdAndIsAbandonedFalse(long id);
 
+  Release findByReleaseKey(String releaseKey);
+
   List<Release> findByAppIdAndClusterNameAndNamespaceNameOrderByIdDesc(String appId, String clusterName, String namespaceName, Pageable page);
 
   List<Release> findByAppIdAndClusterNameAndNamespaceNameAndIsAbandonedFalseOrderByIdDesc(String appId, String clusterName, String namespaceName, Pageable page);
 
   List<Release> findByAppIdAndClusterNameAndNamespaceNameAndIsAbandonedFalseAndIdBetweenOrderByIdDesc(String appId, String clusterName, String namespaceName, long fromId, long toId);
 
-  List<Release> findByReleaseKeyIn(Set<String> releaseKey);
+  List<Release> findByReleaseKeyIn(Set<String> releaseKeys);
 
   List<Release> findByIdIn(Set<Long> releaseIds);
 

apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java

@@ -119,6 +119,10 @@ public List<Release> findByReleaseKeys(Set<String> releaseKeys) {
     return releaseRepository.findByReleaseKeyIn(releaseKeys);
   }
 
+  public Release findByReleaseKey(String releaseKey) {
+    return releaseRepository.findByReleaseKey(releaseKey);
+  }
+
   public Release findLatestActiveRelease(Namespace namespace) {
     return findLatestActiveRelease(namespace.getAppId(),
                                    namespace.getClusterName(), namespace.getNamespaceName());

apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java

@@ -31,6 +31,8 @@
 import com.ctrip.framework.apollo.configservice.service.config.ConfigService;
 import com.ctrip.framework.apollo.configservice.service.config.ConfigServiceWithCache;
 import com.ctrip.framework.apollo.configservice.service.config.DefaultConfigService;
+import com.ctrip.framework.apollo.configservice.service.config.DefaultIncrementalSyncService;
+import com.ctrip.framework.apollo.configservice.service.config.IncrementalSyncService;
 import com.ctrip.framework.apollo.configservice.util.AccessKeyUtil;
 import io.micrometer.core.instrument.MeterRegistry;
 import org.springframework.boot.web.servlet.FilterRegistrationBean;
@@ -69,13 +71,18 @@ public GrayReleaseRulesHolder grayReleaseRulesHolder() {
 
   @Bean
   public ConfigService configService() {
+    // enable local cache
     if (bizConfig.isConfigServiceCacheEnabled()) {
       return new ConfigServiceWithCache(releaseService, releaseMessageService,
           grayReleaseRulesHolder(), bizConfig, meterRegistry);
     }
     return new DefaultConfigService(releaseService, grayReleaseRulesHolder());
   }
 
+  @Bean
+  public IncrementalSyncService incrementalSyncService() {
+    return new DefaultIncrementalSyncService();
+  }
   @Bean
   public static NoOpPasswordEncoder passwordEncoder() {
     return (NoOpPasswordEncoder) NoOpPasswordEncoder.getInstance();

apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java

@@ -16,22 +16,29 @@
  */
 package com.ctrip.framework.apollo.configservice.controller;
 
+import com.ctrip.framework.apollo.biz.config.BizConfig;
 import com.ctrip.framework.apollo.biz.entity.Release;
 import com.ctrip.framework.apollo.common.entity.AppNamespace;
 import com.ctrip.framework.apollo.common.utils.WebUtils;
 import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache;
 import com.ctrip.framework.apollo.configservice.service.config.ConfigService;
+import com.ctrip.framework.apollo.configservice.service.config.IncrementalSyncService;
 import com.ctrip.framework.apollo.configservice.util.InstanceConfigAuditUtil;
 import com.ctrip.framework.apollo.configservice.util.NamespaceUtil;
 import com.ctrip.framework.apollo.core.ConfigConsts;
 import com.ctrip.framework.apollo.core.dto.ApolloConfig;
 import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages;
+import com.ctrip.framework.apollo.core.dto.ConfigurationChange;
+import com.ctrip.framework.apollo.core.enums.ConfigSyncType;
 import com.ctrip.framework.apollo.tracer.Tracer;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -42,6 +49,9 @@
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -55,25 +65,33 @@
 public class ConfigController {
 
   private final ConfigService configService;
+
+  private final IncrementalSyncService incrementalSyncService;
   private final AppNamespaceServiceWithCache appNamespaceService;
   private final NamespaceUtil namespaceUtil;
   private final InstanceConfigAuditUtil instanceConfigAuditUtil;
   private final Gson gson;
+  private final BizConfig bizConfig;
+
 
   private static final Type configurationTypeReference = new TypeToken<Map<String, String>>() {
-      }.getType();
+  }.getType();
 
   public ConfigController(
       final ConfigService configService,
+      final IncrementalSyncService incrementalSyncService,
       final AppNamespaceServiceWithCache appNamespaceService,
       final NamespaceUtil namespaceUtil,
       final InstanceConfigAuditUtil instanceConfigAuditUtil,
-      final Gson gson) {
+      final Gson gson,
+      final BizConfig bizConfig) {
     this.configService = configService;
+    this.incrementalSyncService = incrementalSyncService;
     this.appNamespaceService = appNamespaceService;
     this.namespaceUtil = namespaceUtil;
     this.instanceConfigAuditUtil = instanceConfigAuditUtil;
     this.gson = gson;
+    this.bizConfig = bizConfig;
   }
 
   @GetMapping(value = "/{appId}/{clusterName}/{namespace:.+}")
@@ -132,10 +150,10 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String
 
     auditReleases(appId, clusterName, dataCenter, clientIp, releases);
 
-    String mergedReleaseKey = releases.stream().map(Release::getReleaseKey)
-            .collect(Collectors.joining(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR));
+    String latestMergedReleaseKey = releases.stream().map(Release::getReleaseKey)
+        .collect(Collectors.joining(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR));
 
-    if (mergedReleaseKey.equals(clientSideReleaseKey)) {
+    if (latestMergedReleaseKey.equals(clientSideReleaseKey)) {
       // Client side configuration is the same with server side, return 304
       response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
       Tracer.logEvent("Apollo.Config.NotModified",
@@ -144,8 +162,51 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String
     }
 
     ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace,
-        mergedReleaseKey);
-    apolloConfig.setConfigurations(mergeReleaseConfigurations(releases));
+        latestMergedReleaseKey);
+
+    Map<String, String> latestConfigurations = mergeReleaseConfigurations(releases);
+
+    try {
+      if (bizConfig.isConfigServiceIncrementalChangeEnabled()) {
+        LinkedHashSet<String> clientSideReleaseKeys = Sets.newLinkedHashSet(
+            Arrays.stream(
+                    clientSideReleaseKey.split(Pattern.quote(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR)))
+                .collect(Collectors.toList()));
+
+        Map<String, Release> clientSideReleases = configService.findReleasesByReleaseKeys(
+            clientSideReleaseKeys);
+        //find history releases
+        if (clientSideReleases != null) {
+          //order by clientSideReleaseKeys
+          List<Release> historyReleasesWithOrder = new ArrayList<>();
+          for (String item : clientSideReleaseKeys) {
+            Release release = clientSideReleases.get(item);
+            if (release != null) {
+              historyReleasesWithOrder.add(release);
+            }
+          }
+
+          Map<String, String> clientSideConfigurations = mergeReleaseConfigurations
+              (historyReleasesWithOrder);
+
+          List<ConfigurationChange> configurationChanges = incrementalSyncService.getConfigurationChanges(
+              latestMergedReleaseKey,
+              latestConfigurations, clientSideReleaseKey, clientSideConfigurations);
+
+          apolloConfig.setConfigurationChanges(configurationChanges);
+
+          apolloConfig.setConfigSyncType(ConfigSyncType.INCREMENTAL_SYNC.getValue());
+          Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded,
+              originalNamespace, dataCenter));
+          return apolloConfig;
+        }
+
+      }
+    } catch (ExecutionException e) {
+      //fallback to full sync
+    }
+
+    apolloConfig.setConfigurations(latestConfigurations);
 
     Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded,
         originalNamespace, dataCenter));

apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java

@@ -19,6 +19,9 @@
 import com.ctrip.framework.apollo.biz.entity.Release;
 import com.ctrip.framework.apollo.biz.message.ReleaseMessageListener;
 import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
 
 /**
  * @author Jason Song(song_s@ctrip.com)
@@ -40,4 +43,11 @@ public interface ConfigService extends ReleaseMessageListener {
    */
   Release loadConfig(String clientAppId, String clientIp, String clientLabel, String configAppId, String
       configClusterName, String configNamespace, String dataCenter, ApolloNotificationMessages clientMessages);
+
+
+  /**
+   * @param releaseKeys
+   * @return the ReleaseMap
+   */
+  Map<String, Release> findReleasesByReleaseKeys(Set<String> releaseKeys) throws ExecutionException;
 }