JS-372 Introduce caches to improve SonarLint performance

.cirrus.yml

@@ -129,8 +129,6 @@
   build_and_deploy_script:
     - source cirrus-env BUILD
     - node --version
-    - npm ci
-    - npm run check-format
     - npm run build
     - regular_mvn_build_deploy_analyze -Dmaven.test.skip=true -Dmaven.install.skip=true -Dsonar.skip=true
   cleanup_before_cache_script: cleanup_maven_repository
@@ -393,7 +391,7 @@
     path: 'its/perf/target/perf.txt'
   cleanup_before_cache_script: cleanup_maven_repository
 
 promote_task:
   depends_on:
     - ws_scan
     - build_win

.cirrus/nodejs.Dockerfile

@@ -6,7 +6,7 @@ USER root
 ARG NODE_VERSION=18
 
 RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \
-    && apt-get install -y nodejs=${NODE_VERSION}.* \
+    && apt-get install -y --allow-downgrades nodejs=${NODE_VERSION}.* \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 USER sonarsource

sonar-plugin/bridge/src/main/java/org/sonar/plugins/javascript/bridge/TsConfigFile.java

@@ -19,32 +19,19 @@
  */
 package org.sonar.plugins.javascript.bridge;
 
-import static java.util.Collections.emptyList;
-
 import java.io.IOException;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonar.api.batch.fs.InputFile;
 
-public class TsConfigFile implements Predicate<InputFile> {
+public class TsConfigFile {
 
   private static final Logger LOG = LoggerFactory.getLogger(TsConfigFile.class);
 
-  public static final TsConfigFile UNMATCHED_CONFIG = new TsConfigFile(
-    "NO_CONFIG",
-    emptyList(),
-    emptyList()
-  );
-
   final String filename;
   final Set<String> files;
   final List<String> projectReferences;
@@ -55,17 +42,11 @@ public TsConfigFile(String filename, List<String> files, List<String> projectRef
     this.projectReferences = projectReferences;
   }
 
-  static String normalizePath(String path) {
-    try {
-      return Path
-        .of(path)
-        .toRealPath(LinkOption.NOFOLLOW_LINKS)
-        .toString()
-        .replaceAll("[\\\\/]", "/");
-    } catch (IOException e) {
-      LOG.debug("Could not normalize {}", path);
-      return path;
-    }
+  public static String normalizePath(String path) {
+    return Path
+      .of(path)
+      .toString()
+      .replaceAll("[\\\\/]", "/");
   }
 
   public List<String> getProjectReferences() {
@@ -76,27 +57,8 @@ public String getFilename() {
     return filename;
   }
 
-  @Override
-  public boolean test(InputFile inputFile) {
-    var path = normalizePath(inputFile.absolutePath());
-    return files.contains(path);
-  }
-
-  public static Map<TsConfigFile, List<InputFile>> inputFilesByTsConfig(
-    List<TsConfigFile> tsConfigFiles,
-    List<InputFile> inputFiles
-  ) {
-    Map<TsConfigFile, List<InputFile>> result = new LinkedHashMap<>();
-    inputFiles.forEach(inputFile -> {
-      TsConfigFile tsconfig = tsConfigFiles
-        .stream()
-        .filter(tsConfigFile -> tsConfigFile.test(inputFile))
-        .findFirst()
-        .orElse(UNMATCHED_CONFIG);
-      LOG.debug("{} matched {}", inputFile, tsconfig);
-      result.computeIfAbsent(tsconfig, t -> new ArrayList<>()).add(inputFile);
-    });
-    return result;
+  public Set<String> getFiles() {
+    return files;
   }
 
   @Override

sonar-plugin/sonar-javascript-plugin/src/main/java/org/sonar/plugins/javascript/JavaScriptFilePredicate.java

@@ -101,4 +101,8 @@ private static boolean isVueTsFile(InputFile file) {
   public static boolean isTypeScriptFile(InputFile file) {
     return (TypeScriptLanguage.KEY.equals(file.language()) || isVueTsFile(file));
   }
+
+  public static boolean isJavaScriptFile(InputFile file) {
+    return JavaScriptLanguage.KEY.equals(file.language());
+  }
 }

sonar-plugin/sonar-javascript-plugin/src/main/java/org/sonar/plugins/javascript/JavaScriptPlugin.java

@@ -39,6 +39,7 @@
 import org.sonar.plugins.javascript.analysis.HtmlSensor;
 import org.sonar.plugins.javascript.analysis.JsTsChecks;
 import org.sonar.plugins.javascript.analysis.JsTsSensor;
+import org.sonar.plugins.javascript.analysis.TsConfigCacheImpl;
 import org.sonar.plugins.javascript.analysis.TsConfigProvider;
 import org.sonar.plugins.javascript.analysis.YamlSensor;
 import org.sonar.plugins.javascript.bridge.AnalysisWarningsWrapper;
@@ -323,7 +324,7 @@ public void define(Context context) {
       );
     } else {
       var sonarLintPluginAPIManager = new SonarLintPluginAPIManager();
-      sonarLintPluginAPIManager.addSonarLintTypeCheckingChecker(
+      sonarLintPluginAPIManager.addSonarLintExtensions(
         context,
         new SonarLintPluginAPIVersion()
       );
@@ -332,14 +333,15 @@ public void define(Context context) {
 
   static class SonarLintPluginAPIManager {
 
-    public void addSonarLintTypeCheckingChecker(
+    public void addSonarLintExtensions(
       Context context,
       SonarLintPluginAPIVersion sonarLintPluginAPIVersion
     ) {
       if (sonarLintPluginAPIVersion.isDependencyAvailable()) {
         context.addExtension(SonarLintTypeCheckingCheckerImpl.class);
+        context.addExtension(TsConfigCacheImpl.class);
       } else {
-        LOG.debug("Error while trying to inject SonarLintTypeCheckingChecker");
+        LOG.debug("Error while trying to inject SonarLint extensions");
       }
     }
   }
@@ -348,7 +350,7 @@ static class SonarLintPluginAPIVersion {
 
     boolean isDependencyAvailable() {
       try {
-        Class.forName("org.sonarsource.sonarlint.plugin.api.module.file.ModuleFileSystem");
+        Class.forName("org.sonarsource.sonarlint.plugin.api.module.file.ModuleFileListener");
       } catch (ClassNotFoundException e) {
         return false;
       }

sonar-plugin/sonar-javascript-plugin/src/main/java/org/sonar/plugins/javascript/analysis/AnalysisWithWatchProgram.java

@@ -20,63 +20,41 @@
 package org.sonar.plugins.javascript.analysis;
 
 import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
 import javax.annotation.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.scanner.ScannerSide;
 import org.sonar.plugins.javascript.bridge.AnalysisWarningsWrapper;
 import org.sonar.plugins.javascript.bridge.BridgeServer;
-import org.sonar.plugins.javascript.bridge.TsConfigFile;
 import org.sonar.plugins.javascript.utils.ProgressReport;
 import org.sonarsource.api.sonarlint.SonarLintSide;
 
 @ScannerSide
 @SonarLintSide
 public class AnalysisWithWatchProgram extends AbstractAnalysis {
 
-  private static final Logger LOG = LoggerFactory.getLogger(AnalysisWithWatchProgram.class);
+  TsConfigCache tsConfigCache;
 
   public AnalysisWithWatchProgram(
     BridgeServer bridgeServer,
     AnalysisProcessor analysisProcessor,
-    AnalysisWarningsWrapper analysisWarnings
+    AnalysisWarningsWrapper analysisWarnings,
+    @Nullable TsConfigCache tsConfigCache
   ) {
     super(bridgeServer, analysisProcessor, analysisWarnings);
+    this.tsConfigCache = tsConfigCache;
   }
 
   @Override
   public void analyzeFiles(List<InputFile> inputFiles, List<String> tsConfigs) throws IOException {
     boolean success = false;
     progressReport = new ProgressReport(PROGRESS_REPORT_TITLE, PROGRESS_REPORT_PERIOD);
-    Map<TsConfigFile, List<InputFile>> filesByTsConfig = TsConfigFile.inputFilesByTsConfig(
-      loadTsConfigs(tsConfigs),
-      inputFiles
-    );
     try {
       progressReport.start(inputFiles.size(), inputFiles.iterator().next().toString());
-      if (tsConfigs.isEmpty()) {
-        LOG.info("Analyzing {} files without tsconfig", inputFiles.size());
-        analyzeTsConfig(null, inputFiles);
-      } else {
-        for (Map.Entry<TsConfigFile, List<InputFile>> entry : filesByTsConfig.entrySet()) {
-          TsConfigFile tsConfigFile = entry.getKey();
-          List<InputFile> files = entry.getValue();
-          if (TsConfigFile.UNMATCHED_CONFIG.equals(tsConfigFile)) {
-            LOG.info("Analyzing {} files without tsconfig", files.size());
-            analyzeTsConfig(null, files);
-          } else {
-            LOG.info("Analyzing {} files using tsconfig: {}", files.size(), tsConfigFile);
-            analyzeTsConfig(tsConfigFile, files);
-          }
-        }
+      for (InputFile inputFile : inputFiles) {
+        var tsConfigFile = tsConfigCache.getTsConfigForInputFile(inputFile);
+        // process
+        analyzeFile(inputFile, tsConfigFile == null ? List.of() : List.of(tsConfigFile.getFilename()), null);
       }
       success = true;
       if (analysisProcessor.parsingErrorFilesCount() > 0) {
@@ -95,30 +73,4 @@ public void analyzeFiles(List<InputFile> inputFiles, List<String> tsConfigs) thr
       }
     }
   }
-
-  private List<TsConfigFile> loadTsConfigs(List<String> tsConfigPaths) {
-    List<TsConfigFile> tsConfigFiles = new ArrayList<>();
-    Deque<String> workList = new ArrayDeque<>(tsConfigPaths);
-    Set<String> processed = new HashSet<>();
-    while (!workList.isEmpty()) {
-      String path = workList.pop();
-      if (processed.add(path)) {
-        TsConfigFile tsConfigFile = bridgeServer.loadTsConfig(path);
-        tsConfigFiles.add(tsConfigFile);
-        if (!tsConfigFile.getProjectReferences().isEmpty()) {
-          LOG.debug("Adding referenced project's tsconfigs {}", tsConfigFile.getProjectReferences());
-        }
-        workList.addAll(tsConfigFile.getProjectReferences());
-      }
-    }
-    return tsConfigFiles;
-  }
-
-  private void analyzeTsConfig(@Nullable TsConfigFile tsConfigFile, List<InputFile> files)
-    throws IOException {
-    List<String> tsConfigs = tsConfigFile == null ? List.of() : List.of(tsConfigFile.getFilename());
-    for (InputFile inputFile : files) {
-      analyzeFile(inputFile, tsConfigs, null);
-    }
-  }
 }

sonar-plugin/sonar-javascript-plugin/src/main/java/org/sonar/plugins/javascript/analysis/JsTsSensor.java

@@ -19,6 +19,8 @@
  */
 package org.sonar.plugins.javascript.analysis;
 
+import static org.sonar.plugins.javascript.analysis.TsConfigProvider.getTsConfigs;
+
 import java.io.IOException;
 import java.util.List;
 import java.util.stream.StreamSupport;
@@ -46,6 +48,7 @@ public class JsTsSensor extends AbstractBridgeSensor {
   private final JsTsChecks checks;
   private final SonarLintTypeCheckingChecker javaScriptProjectChecker;
   private final AnalysisConsumers consumers;
+  private final TsConfigCache tsConfigCache;
 
   // Constructor for SonarCloud without the optional dependency (Pico doesn't support optional dependencies)
   public JsTsSensor(
@@ -79,6 +82,7 @@ public JsTsSensor(
     this.checks = checks;
     this.javaScriptProjectChecker = javaScriptProjectChecker;
     this.consumers = consumers;
+    this.tsConfigCache = analysisWithWatchProgram.tsConfigCache;
   }
 
   @Override
@@ -110,10 +114,11 @@ protected void analyzeFiles(List<InputFile> inputFiles) throws IOException {
     );
 
     SonarLintTypeCheckingChecker.checkOnce(javaScriptProjectChecker, context);
-    var tsConfigs = TsConfigProvider.getTsConfigs(
+    var tsConfigs = getTsConfigs(
       contextUtils,
       javaScriptProjectChecker,
-      this::createTsConfigFile
+      this::createTsConfigFile,
+      tsConfigCache
     );
     AbstractAnalysis analysis;
     if (shouldAnalyzeWithProgram()) {