Skip to content

Commit

Permalink
Go With Replace Detector no File Parsing (#1181)
Browse files Browse the repository at this point in the history 
* not parsing for replace module

* require

---------

Co-authored-by: Amitla Vannikumar <[email protected]>
  • Loading branch information
@amitla1
amitla1 and Amitla Vannikumar authored Jun 19, 2024
1 parent dafe527 commit 2284e06
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 146 deletions.
106 changes: 20 additions & 86 deletions src/Microsoft.ComponentDetection.Detectors/go/GoComponentWithReplaceDetector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ namespace Microsoft.ComponentDetection.Detectors.Go;
using Microsoft.ComponentDetection.Contracts.Internal;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.Extensions.Logging;
using MoreLinq;
using Newtonsoft.Json;

public class GoComponentWithReplaceDetector : FileComponentDetector, IExperimentalDetector
Expand Down Expand Up @@ -257,7 +256,7 @@ private async Task<bool> UseGoCliToScanAsync(string location, ISingleFileCompone
return true;
}

private void AddGoComponent(IList<GoComponent> goComponents, string line, ISingleFileComponentRecorder singleFileComponentRecorder)
private void TryRegisterDependencyFromModLine(string line, ISingleFileComponentRecorder singleFileComponentRecorder)
{
if (line.Trim().StartsWith("//"))
{
Expand All @@ -267,7 +266,7 @@ private void AddGoComponent(IList<GoComponent> goComponents, string line, ISingl

if (this.TryToCreateGoComponentFromModLine(line, out var goComponent))
{
goComponents.Add(goComponent);
singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent));
}
else
{
Expand All @@ -277,90 +276,42 @@ private void AddGoComponent(IList<GoComponent> goComponents, string line, ISingl
}
}

private void ReplaceGoComponents(IList<GoComponent> goComponents, string line, ISingleFileComponentRecorder singleFileComponentRecorder)
{
if (this.TryToCreateReplacementGoComponentFromModLine(line, out var goComponent))
{
var goComponentsWithReplacementVersion = goComponents.Where(component => component.Name == goComponent.Name);
if (goComponentsWithReplacementVersion.Any())
{
foreach (var component in goComponentsWithReplacementVersion)
{
if (component.Name == goComponent.Name)
{
component.Version = goComponent.Version;
}
}
}
}
else
{
var lineTrim = line.Trim();
this.Logger.LogWarning("Line could not be parsed for component [{LineTrim}]", lineTrim);
singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim);
}
}

private void TryRegisterDependencyFromModLine(IList<GoComponent> goComponents, ISingleFileComponentRecorder singleFileComponentRecorder)
{
goComponents.ForEach(goComponent => singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent)));
}

private async Task ParseGoModFileAsync(
ISingleFileComponentRecorder singleFileComponentRecorder,
IComponentStream file,
GoGraphTelemetryRecord goGraphTelemetryRecord)
{
using var reader = new StreamReader(file.Stream);
var goComponents = new List<GoComponent>();
var inRequireBlock = false;
var inReplaceBlock = false;
var startString = "require ";

// There can be multiple require( ) sections in go 1.17+. loop over all of them.
while (!reader.EndOfStream)
{
var line = await reader.ReadLineAsync();
if (line != null && line.StartsWith("require ("))
{
inRequireBlock = true;
continue;
}
else if (line != null && line.StartsWith("replace ("))
{
inReplaceBlock = true;
continue;
}
else if (line != null && line.StartsWith(")"))
{
inRequireBlock = false;
inReplaceBlock = false;
continue;
}

if (line != null && line.StartsWith("require "))
{
this.AddGoComponent(goComponents, line[8..], singleFileComponentRecorder);
}
else if (line != null && line.StartsWith("replace "))
while (line != null && !line.StartsWith("require ("))
{
this.ReplaceGoComponents(goComponents, line[8..], singleFileComponentRecorder);
}
if (line.StartsWith("go "))
{
goGraphTelemetryRecord.GoModVersion = line[3..].Trim();
}

if (inRequireBlock)
{
this.AddGoComponent(goComponents, line, singleFileComponentRecorder);
}
else if (inReplaceBlock)
{
this.ReplaceGoComponents(goComponents, line, singleFileComponentRecorder);
// In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies
// are listed in the require () section
if (line.StartsWith(startString))
{
this.TryRegisterDependencyFromModLine(line[startString.Length..], singleFileComponentRecorder);
}

line = await reader.ReadLineAsync();
}
else if (line != null && line.StartsWith("go "))

// Stopping at the first ) restrict the detection to only the require section.
while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(")"))
{
goGraphTelemetryRecord.GoModVersion = line[3..].Trim();
this.TryRegisterDependencyFromModLine(line, singleFileComponentRecorder);
}
}

this.TryRegisterDependencyFromModLine(goComponents, singleFileComponentRecorder);
}

private bool TryToCreateGoComponentFromModLine(string line, out GoComponent goComponent)
Expand All @@ -380,23 +331,6 @@ private bool TryToCreateGoComponentFromModLine(string line, out GoComponent goCo
return true;
}

private bool TryToCreateReplacementGoComponentFromModLine(string line, out GoComponent goComponent)
{
var lineComponents = Regex.Split(line.Trim(), @"\s+");

if (lineComponents.Length < 2)
{
goComponent = null;
return false;
}

var name = lineComponents[2];
var version = lineComponents[3];
goComponent = new GoComponent(name, version);

return true;
}

// For more information about the format of the go.sum file
// visit https://golang.org/cmd/go/#hdr-Module_authentication_using_go_sum
private void ParseGoSumFile(
Expand Down
60 changes: 0 additions & 60 deletions test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentWithReplaceDetectorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,66 +315,6 @@ public async Task TestGoModDetector_DetectorOnlyDetectInsideRequireSectionAsync(
discoveredComponents.Where(component => component.Component.Id == "github.com/kr/pretty v0.1.0 - Go").Should().ContainSingle();
}

[TestMethod]
public async Task TestGoModDetector_DetectReplaceSectionAsync()
{
var goMod =
@"module github.com/Azure/azure-storage-blob-go
require (
github.com/Azure/azure-pipeline-go v0.2.1
github.com/docker/distribution v2.7.1+incompatible
github.com/Masterminds/sprig/v3 v3.1.0
)
replace (
github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.2+incompatible
github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d
)
";
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.mod", goMod)
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);

var detectedComponents = componentRecorder.GetDetectedComponents();
detectedComponents.Should().HaveCount(3);

var discoveredComponents = detectedComponents.ToArray();
discoveredComponents.Where(component => component.Component.Id == "github.com/Azure/azure-pipeline-go v0.2.1 - Go").Should().ContainSingle();
discoveredComponents.Where(component => component.Component.Id == "github.com/Masterminds/sprig/v3 v3.1.0 - Go").Should().ContainSingle();
discoveredComponents.Where(component => component.Component.Id == "github.com/docker/distribution v0.0.0-20191216044856-a8371794149d - Go").Should().ContainSingle();
}

[TestMethod]
public async Task TestGoModDetector_SkipsGoSumFilesWithReplaceAsync()
{
var goMod =
@"module contoso.com/greetings
go 1.18
require github.com/go-sql-driver/mysql v1.7.1 // indirect
replace github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.3";

var goSum =
@"github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=";

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile("go.mod", goMod)
.WithFile("go.mod", goMod, new[] { "go.mod" })
.WithFile("go.sum", goSum)
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
componentRecorder.GetDetectedComponents().Should().ContainSingle();

var component = componentRecorder.GetDetectedComponents().First();
component.Component.Id.Should().Be("github.com/go-sql-driver/mysql v1.7.3 - Go");
}

[TestMethod]
public async Task TestGoDetector_GoCommandNotFoundAsync()
{
Expand Down

0 comments on commit 2284e06

Please sign in to comment.