Merge pull request anchore#150 from anchore/issue-59

Add a `setup.py` cataloger

syft/cataloger/python/cataloger.go

@@ -22,6 +22,7 @@ func New() *Cataloger {
 		"**/*dist-info/METADATA": parseWheelMetadata,
 		"**/requirements.txt":    parseRequirementsTxt,
 		"**/poetry.lock":         parsePoetryLock,
+		"**/setup.py":            parseSetup,
 	}
 
 	return &Cataloger{

syft/cataloger/python/parse_setup.go

@@ -0,0 +1,50 @@
+package python
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/anchore/syft/syft/cataloger/common"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// integrity check
+var _ common.ParserFn = parseSetup
+
+// match examples:
+//		'pathlib3==2.2.0;python_version<"3.6"'  --> match(name=pathlib3 version=2.2.0)
+//		 "mypy==v0.770",                        --> match(name=mypy version=v0.770)
+//		" mypy2 == v0.770", ' mypy3== v0.770',  --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
+var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
+
+func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
+	packages := make([]pkg.Package, 0)
+
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		line = strings.TrimRight(line, "\n")
+
+		for _, match := range pinnedDependency.FindAllString(line, -1) {
+			parts := strings.Split(match, "==")
+			if len(parts) != 2 {
+				continue
+			}
+			name := strings.Trim(parts[0], "'\"")
+			name = strings.TrimSpace(name)
+
+			version := strings.TrimSpace(parts[len(parts)-1])
+			packages = append(packages, pkg.Package{
+				Name:     strings.Trim(name, "'\""),
+				Version:  strings.Trim(version, "'\""),
+				Language: pkg.Python,
+				Type:     pkg.PythonSetupPkg,
+			})
+		}
+	}
+
+	return packages, nil
+}

syft/cataloger/python/parse_setup_test.go

@@ -0,0 +1,60 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func TestParseSetup(t *testing.T) {
+	expected := map[string]pkg.Package{
+		"pathlib3": {
+			Name:     "pathlib3",
+			Version:  "2.2.0",
+			Language: pkg.Python,
+			Type:     pkg.PythonSetupPkg,
+			Licenses: []string{},
+		},
+		"mypy": {
+			Name:     "mypy",
+			Version:  "v0.770",
+			Language: pkg.Python,
+			Type:     pkg.PythonSetupPkg,
+			Licenses: []string{},
+		},
+		"mypy1": {
+			Name:     "mypy1",
+			Version:  "v0.770",
+			Language: pkg.Python,
+			Type:     pkg.PythonSetupPkg,
+			Licenses: []string{},
+		},
+		"mypy2": {
+			Name:     "mypy2",
+			Version:  "v0.770",
+			Language: pkg.Python,
+			Type:     pkg.PythonSetupPkg,
+			Licenses: []string{},
+		},
+		"mypy3": {
+			Name:     "mypy3",
+			Version:  "v0.770",
+			Language: pkg.Python,
+			Type:     pkg.PythonSetupPkg,
+			Licenses: []string{},
+		},
+	}
+	fixture, err := os.Open("test-fixtures/setup/setup.py")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseSetup(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse requirements: %+v", err)
+	}
+
+	assertPkgsEqual(t, actual, expected)
+
+}

syft/cataloger/python/test-fixtures/setup/setup.py

@@ -0,0 +1,46 @@
+from setuptools import setup
+
+# Sample setup.py from the pytest project with added comments specific
+# to the cataloger
+
+INSTALL_REQUIRES = [
+    "py>=1.5.0",
+    "packaging",
+    "attrs>=17.4.0",
+    "more-itertools>=4.0.0",
+    'atomicwrites>=1.0;sys_platform=="win32"',  # sys_platform is ignored
+    'pathlib2>=2.2.0;python_version=="3.6"',  # python_version is ignored
+    'pathlib3==2.2.0;python_version<"3.6"',  # this is caught
+    'colorama;sys_platform=="win32"',
+    "pluggy>=0.12,<1.0",
+    'importlib-metadata>=0.12;python_version<"3.8"',
+    "wcwidth",
+]
+
+
+def main():
+    setup(
+        use_scm_version={"write_to": "src/_pytest/_version.py"},
+        setup_requires=["setuptools-scm", "setuptools>=40.0"],
+        package_dir={"": "src"},
+        extras_require={
+            "testing": [
+                "argcomplete",
+                "hypothesis>=3.56",
+                "mock",
+                "nose",
+                "requests",
+                "xmlschema",
+            ],
+            "checkqa-mypy": [
+                "mypy==v0.770",  # this is caught
+                " mypy1==v0.770",  # this is caught
+                " mypy2 == v0.770", ' mypy3== v0.770',  # this is caught
+            ],
+        },
+        install_requires=INSTALL_REQUIRES,
+    )
+
+
+if __name__ == "__main__":
+    main()

syft/pkg/type.go

@@ -16,6 +16,7 @@ const (
 	NpmPkg                Type = "npm"
 	YarnPkg               Type = "yarn"
 	PythonRequirementsPkg Type = "python-requirements"
+	PythonSetupPkg        Type = "python-setup"
 	JavaPkg               Type = "java-archive"
 	JenkinsPluginPkg      Type = "jenkins-plugin"
 	GoModulePkg           Type = "go-module"
@@ -32,6 +33,7 @@ var AllPkgs = []Type{
 	NpmPkg,
 	YarnPkg,
 	PythonRequirementsPkg,
+	PythonSetupPkg,
 	JavaPkg,
 	JenkinsPluginPkg,
 	GoModulePkg,

test/integration/pkg_cases.go

@@ -77,13 +77,21 @@ var cases = []struct {
 		},
 	},
 	{
-		name:        "find python packages",
+		name:        "find python requirements.txt packages",
 		pkgType:     pkg.PythonRequirementsPkg,
 		pkgLanguage: pkg.Python,
 		pkgInfo: map[string]string{
 			"flask": "4.0.0",
 		},
 	},
+	{
+		name:        "find python setup.py packages",
+		pkgType:     pkg.PythonSetupPkg,
+		pkgLanguage: pkg.Python,
+		pkgInfo: map[string]string{
+			"mypy": "v0.770",
+		},
+	},
 	{
 		name:        "find bundler packages",
 		pkgType:     pkg.BundlerPkg,

test/integration/test-fixtures/image-pkg-coverage/python/setup/setup.py

@@ -0,0 +1,42 @@
+from setuptools import setup
+
+# TODO: if py gets upgrade to >=1.6,
+#       remove _width_of_current_line in terminal.py
+INSTALL_REQUIRES = [
+    "py>=1.5.0",
+    "packaging",
+    "attrs>=17.4.0",  # should match oldattrs tox env.
+    "more-itertools>=4.0.0",
+    'atomicwrites>=1.0;sys_platform=="win32"',
+    'pathlib2>=2.2.0;python_version<"3.6"',
+    'colorama;sys_platform=="win32"',
+    "pluggy>=0.12,<1.0",
+    'importlib-metadata>=0.12;python_version<"3.8"',
+    "wcwidth",
+]
+
+
+def main():
+    setup(
+        use_scm_version={"write_to": "src/_pytest/_version.py"},
+        setup_requires=["setuptools-scm", "setuptools>=40.0"],
+        package_dir={"": "src"},
+        extras_require={
+            "testing": [
+                "argcomplete",
+                "hypothesis>=3.56",
+                "mock",
+                "nose",
+                "requests",
+                "xmlschema",
+            ],
+            "checkqa-mypy": [
+                "mypy==v0.770",  # keep this in sync with .pre-commit-config.yaml.
+            ],
+        },
+        install_requires=INSTALL_REQUIRES,
+    )
+
+
+if __name__ == "__main__":
+    main()