Golang: Display Go external libraries based on importpath structure

base/src/com/google/idea/blaze/base/sync/libraries/BlazeExternalLibraryProvider.java

@@ -34,7 +34,7 @@ protected abstract ImmutableList<File> getLibraryFiles(
       Project project, BlazeProjectData projectData);
 
   @Override
-  public final Collection<SyntheticLibrary> getAdditionalProjectLibraries(Project project) {
+  public Collection<SyntheticLibrary> getAdditionalProjectLibraries(Project project) {
     SyntheticLibrary library = ExternalLibraryManager.getInstance(project).getLibrary(getClass());
     return library != null ? ImmutableList.of(library) : ImmutableList.of();
   }

base/src/com/google/idea/blaze/base/sync/libraries/BlazeExternalSyntheticLibrary.java

@@ -36,7 +36,7 @@
  * A {@link SyntheticLibrary} pointing to a list of external files for a language. Only supports one
  * instance per value of presentableText.
  */
-public final class BlazeExternalSyntheticLibrary extends SyntheticLibrary
+public class BlazeExternalSyntheticLibrary extends SyntheticLibrary
     implements ItemPresentation {
   private final String presentableText;
   private final ImmutableSet<File> files;
@@ -49,7 +49,7 @@ public final class BlazeExternalSyntheticLibrary extends SyntheticLibrary
    *     equals, hashcode -- there must only be one instance per value of this text
    * @param files collection of files that this synthetic library is responsible for.
    */
-  BlazeExternalSyntheticLibrary(String presentableText, Collection<File> files) {
+  public BlazeExternalSyntheticLibrary(String presentableText, Collection<File> files) {
     this.presentableText = presentableText;
     this.files = ImmutableSet.copyOf(files);
     this.validFiles =

golang/src/META-INF/go-contents.xml

@@ -44,5 +44,6 @@
     <documentationProvider implementation="com.google.idea.blaze.golang.resolve.BlazeGoImportResolver$GoPackageDocumentationProvider"/>
     <additionalLibraryRootsProvider implementation="com.google.idea.blaze.golang.sync.BlazeGoAdditionalLibraryRootsProvider"/>
     <postStartupActivity implementation="com.google.idea.blaze.golang.run.producers.NonBlazeProducerSuppressor"/>
+    <treeStructureProvider implementation="com.google.idea.blaze.golang.treeview.BlazeGoTreeStructureProvider" order="last"/>
   </extensions>
 </idea-plugin>

golang/src/com/google/idea/blaze/golang/resolve/BlazeGoPackage.java

@@ -41,6 +41,9 @@
 import com.google.idea.blaze.base.model.primitives.Label;
 import com.google.idea.blaze.base.model.primitives.LanguageClass;
 import com.google.idea.blaze.base.model.primitives.RuleType;
+import com.google.idea.blaze.base.model.primitives.WorkspacePath;
+import com.google.idea.blaze.base.model.primitives.WorkspaceRoot;
+import com.google.idea.blaze.base.sync.projectview.ImportRoots;
 import com.google.idea.blaze.base.sync.SyncCache;
 import com.google.idea.blaze.base.sync.workspace.WorkspaceHelper;
 import com.google.idea.blaze.base.targetmaps.ReverseDependencyMap;
@@ -164,6 +167,29 @@ public static ImmutableMultimap<Label, File> getUncachedTargetToFileMap(
     return builder.build();
   }
 
+  public static ImmutableMultimap<String, File> getUncachedExternalImportpathToFilesMap(
+          Project project, BlazeProjectData projectData, ImportRoots importRoots, WorkspaceRoot workspaceRoot) {
+    ImmutableMultimap<Label, GoIdeInfo> libraryToTestMap = buildLibraryToTestMap(projectData);
+    ImmutableMultimap.Builder<String, File> builder = ImmutableMultimap.builder();
+    Predicate<File> isExternal =
+            f -> {
+              WorkspacePath path = workspaceRoot.workspacePathForSafe(f);
+              return path == null || !importRoots.containsWorkspacePath(path);
+            };
+    for (TargetIdeInfo target : projectData.getTargetMap().targets()) {
+      if (target.getGoIdeInfo() == null || target.getGoIdeInfo().getImportPath() == null) {
+        continue;
+      }
+      builder.putAll(
+              target.getGoIdeInfo().getImportPath(),
+              getSourceFiles(target, project, projectData, libraryToTestMap).stream().
+                      filter(f -> f.getName().endsWith(".go")).
+                      filter(isExternal).
+                      collect(toImmutableSet()));
+    }
+    return builder.build();
+  }
+
   private static ImmutableSet<File> getSourceFiles(
       TargetIdeInfo target,
       Project project,

golang/src/com/google/idea/blaze/golang/resolve/BlazeGoPackageFactory.java

@@ -36,7 +36,7 @@
 import java.util.concurrent.ConcurrentMap;
 import javax.annotation.Nullable;
 
-class BlazeGoPackageFactory implements GoPackageFactory {
+public class BlazeGoPackageFactory implements GoPackageFactory {
   @Nullable
   @Override
   public GoPackage createPackage(GoFile goFile) {
@@ -54,7 +54,7 @@ public GoPackage createPackage(GoFile goFile) {
   }
 
   @Nullable
-  static ConcurrentMap<File, String> getFileToImportPathMap(Project project) {
+  public static ConcurrentMap<File, String> getFileToImportPathMap(Project project) {
     return SyncCache.getInstance(project)
         .get(BlazeGoPackageFactory.class, BlazeGoPackageFactory::buildFileToImportPathMap);
   }

golang/src/com/google/idea/blaze/golang/sync/BlazeGoAdditionalLibraryRootsProvider.java

@@ -18,22 +18,25 @@
 import static com.google.common.collect.ImmutableList.toImmutableList;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.idea.blaze.base.model.BlazeProjectData;
 import com.google.idea.blaze.base.model.primitives.LanguageClass;
-import com.google.idea.blaze.base.model.primitives.WorkspacePath;
 import com.google.idea.blaze.base.model.primitives.WorkspaceRoot;
 import com.google.idea.blaze.base.sync.libraries.BlazeExternalLibraryProvider;
 import com.google.idea.blaze.base.sync.projectview.ImportRoots;
 import com.google.idea.blaze.golang.resolve.BlazeGoPackage;
 import com.google.idea.common.experiments.BoolExperiment;
 import com.intellij.openapi.project.Project;
+import com.intellij.openapi.roots.SyntheticLibrary;
 import java.io.File;
-import java.util.function.Predicate;
+import java.util.Collection;
 
 /** Provides out-of-project go sources for indexing. */
 public final class BlazeGoAdditionalLibraryRootsProvider extends BlazeExternalLibraryProvider {
   private static final BoolExperiment useGoAdditionalLibraryRootsProvider =
       new BoolExperiment("use.go.additional.library.roots.provider4", true);
+  private static ImmutableMultimap<String, File> importpathToFilesMap;
+
 
   @Override
   protected String getLibraryName() {
@@ -60,18 +63,17 @@ static ImmutableList<File> getLibraryFiles(
     if (workspaceRoot == null) {
       return ImmutableList.of();
     }
-    Predicate<File> isExternal =
-        f -> {
-          WorkspacePath path = workspaceRoot.workspacePathForSafe(f);
-          return path == null || !importRoots.containsWorkspacePath(path);
-        };
     // don't use sync cache, because
     // 1. this is used during sync before project data is saved
     // 2. the roots provider is its own cache
-    return BlazeGoPackage.getUncachedTargetToFileMap(project, projectData).values().stream()
-        .filter(isExternal)
-        .filter(f -> f.getName().endsWith(".go"))
-        .distinct()
-        .collect(toImmutableList());
+    importpathToFilesMap = BlazeGoPackage.getUncachedExternalImportpathToFilesMap(project, projectData, importRoots, workspaceRoot);
+    return importpathToFilesMap.values().asList();
+  }
+
+  @Override
+  public Collection<SyntheticLibrary> getAdditionalProjectLibraries(Project project) {
+    return importpathToFilesMap != null
+            ? ImmutableList.of(new BlazeGoExternalSyntheticLibrary(getLibraryName(), importpathToFilesMap))
+            : ImmutableList.of();
   }
 }

golang/src/com/google/idea/blaze/golang/sync/BlazeGoExternalSyntheticLibrary.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2021 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.idea.blaze.golang.sync;
+
+import com.goide.GoIcons;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.idea.blaze.base.sync.libraries.BlazeExternalSyntheticLibrary;
+import com.intellij.openapi.roots.SyntheticLibrary;
+import java.io.File;
+import java.util.Collection;
+import javax.swing.Icon;
+
+/**
+ * A {@link BlazeExternalSyntheticLibrary} for Go external libraries. Only supports one instance
+ * per importPath. 
+ */
+public final class BlazeGoExternalSyntheticLibrary extends BlazeExternalSyntheticLibrary {
+    private final ImmutableMultimap<String, File> importpathToFilesMap;
+    private final String importPath;
+    private final boolean isRoot;
+
+    /**
+    * Constructs a root Go external library. Holds a flat list of all external source files. Later
+    * processed by {@link BlazeGoTreeStructureProvider} to structure external source files based on
+    * their importpath.
+    */
+    public BlazeGoExternalSyntheticLibrary(String presentableText, ImmutableMultimap<String, File> importpathToFilesMap) {
+        super(presentableText, importpathToFilesMap.values());
+        this.importpathToFilesMap = importpathToFilesMap;
+        this.importPath = presentableText;
+        this.isRoot = true;
+    }
+
+    /**
+    * Constructs a non-root Go external library. Holds the external source files for an external directory.
+    * Intended for use by {@link GoSyntheticLibraryElementNode}.
+    */
+    public BlazeGoExternalSyntheticLibrary(String presentableText, String importPath, Collection<File> files) {
+        super(presentableText, files);
+        this.importpathToFilesMap = null;
+        this.importPath = importPath;
+        this.isRoot = false;
+    }
+
+    public boolean isRoot() {
+        return this.isRoot;
+    }
+
+    public ImmutableMultimap<String, File> getImportpathToFilesMap() {
+        return importpathToFilesMap;
+    }
+
+    @Override
+    public Icon getIcon(boolean unused) {
+        return GoIcons.VENDOR_DIRECTORY;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        // intended to be only a single instance added to the project for each value of importPath
+        return o instanceof BlazeGoExternalSyntheticLibrary
+            && importPath.equals(((BlazeGoExternalSyntheticLibrary) o).importPath);
+    }
+
+    @Override
+    public int hashCode() {
+        // intended to be only a single instance added to the project for each value of importPath
+        return importPath.hashCode();
+    }
+}

golang/src/com/google/idea/blaze/golang/treeview/BlazeGoTreeStructureProvider.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2021 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.idea.blaze.golang.treeview;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.idea.blaze.base.settings.Blaze;
+import com.google.idea.blaze.base.sync.libraries.BlazeExternalSyntheticLibrary;
+import com.google.idea.blaze.golang.sync.BlazeGoExternalSyntheticLibrary;
+import com.intellij.ide.projectView.impl.nodes.ExternalLibrariesNode;
+import com.intellij.ide.projectView.impl.nodes.SyntheticLibraryElementNode;
+import com.intellij.ide.projectView.TreeStructureProvider;
+import com.intellij.ide.projectView.ViewSettings;
+import com.intellij.ide.util.treeView.AbstractTreeNode;
+import com.intellij.navigation.ItemPresentation;
+import com.intellij.openapi.project.DumbAware;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.roots.SyntheticLibrary;
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Iterator;
+import java.util.HashMap;
+
+/**
+ * Modifies the project view by replacing the External Go Libraries root node (contaning a flat list of sources)
+ * with a root node that structures sources based on their importpaths.
+ */
+public class BlazeGoTreeStructureProvider implements TreeStructureProvider, DumbAware {
+  @Override
+  public Collection<AbstractTreeNode<?>> modify(
+      AbstractTreeNode<?> parent, Collection<AbstractTreeNode<?>> children, ViewSettings settings) {
+    Project project = parent.getProject();
+    if (!Blaze.isBlazeProject(project) || !(parent instanceof ExternalLibrariesNode)) {
+      return children;
+    }
+    List<AbstractTreeNode<?>> newChildren = new ArrayList<AbstractTreeNode<?>>();
+    for (AbstractTreeNode<?> child : children) {
+      if ((child.getValue() instanceof BlazeGoExternalSyntheticLibrary)) {
+        BlazeGoExternalSyntheticLibrary libraryRoot = (BlazeGoExternalSyntheticLibrary)child.getValue();
+        child = libraryRoot.isRoot() ? createSyntheticDirectoryStructure(libraryRoot, settings, project) : child;
+      }
+      newChildren.add(child);
+    }
+    return newChildren;
+  }
+
+  /**
+  * Creates the importpath based project structure.
+  * Loops through the Go project's external importpath references and creates a {@link GoSyntheticLibraryElementNode} for each directory.
+  * Each node holds a {@link BlazeExternalSyntheticLibrary} representing the dir's source files.
+  */
+  private static GoSyntheticLibraryElementNode createSyntheticDirectoryStructure(BlazeGoExternalSyntheticLibrary libraryRoot, ViewSettings settings, Project project) {
+    ImmutableMultimap<String, File> importPathToFilesMap = libraryRoot.getImportpathToFilesMap();
+    BlazeExternalSyntheticLibrary newLibraryRoot =  new BlazeExternalSyntheticLibrary(libraryRoot.getPresentableText(), Collections.emptyList());
+    GoSyntheticLibraryElementNode rootNode = new GoSyntheticLibraryElementNode(project, newLibraryRoot, (ItemPresentation)newLibraryRoot, settings, new HashMap<String, GoSyntheticLibraryElementNode>());
+    String[] imports = importPathToFilesMap.keySet().toArray(String[]::new);
+    // sort imports to ensure parent dirs are visited before children
+    Arrays.sort(imports);
+    for (String key : imports) {
+      GoSyntheticLibraryElementNode currRoot = rootNode;
+      String currImport = null;
+      Iterator<Path> it = Paths.get(key).iterator();
+      while (it.hasNext()) {
+        String dir = it.next().toString();
+        currImport = currImport == null ? dir : currImport + "/" + dir;
+        if (!currRoot.hasChild(currImport)) {
+          Collection<File> files = it.hasNext() ? Collections.emptyList() :  importPathToFilesMap.get(key);
+          BlazeExternalSyntheticLibrary currLib =  new BlazeGoExternalSyntheticLibrary(dir, currImport, files);
+          GoSyntheticLibraryElementNode currNode = new GoSyntheticLibraryElementNode(project, currLib, (ItemPresentation)currLib, settings, new HashMap<String, GoSyntheticLibraryElementNode>());
+          currRoot.addChild(currImport, currNode);
+        }
+        currRoot = currRoot.getChild(currImport);
+      }
+    }
+    return rootNode;
+  }
+}