diff --git a/.automation/test/sample_project_sarif/.eslintrc.json b/.automation/test/sample_project_sarif/.eslintrc.json new file mode 100644 index 00000000000..b69dd82bc9f --- /dev/null +++ b/.automation/test/sample_project_sarif/.eslintrc.json @@ -0,0 +1,13 @@ +{ + "env": { + "node": true, + "commonjs": true, + "es2021": true + }, + "extends": "eslint:recommended", + "parserOptions": { + "ecmaVersion": 12 + }, + "rules": { + } +} diff --git a/.automation/test/sample_project_sarif/gitleaks_bad_01.txt b/.automation/test/sample_project_sarif/gitleaks_bad_01.txt new file mode 100644 index 00000000000..b4b6ca41f00 --- /dev/null +++ b/.automation/test/sample_project_sarif/gitleaks_bad_01.txt @@ -0,0 +1,2 @@ +aws_access_key_id = AROA47DSWDEZA3RQASWB +aws_secret_access_key = wQwdsZDiWg4UA5ngO0OSI2TkM4kkYxF6d2S1aYWM \ No newline at end of file diff --git a/.automation/test/sample_project_sarif/javascript_bad_1.js b/.automation/test/sample_project_sarif/javascript_bad_1.js new file mode 100644 index 00000000000..98e5ee29e32 --- /dev/null +++ b/.automation/test/sample_project_sarif/javascript_bad_1.js @@ -0,0 +1,225 @@ +var http = require('http') +var createHandler = require( 'github-webhook-handler') + +var handler = createHandler( { path : /webhook, secret : (process.env.SECRET) }) + +var userArray = [ 'user1' ] +here is some garbage = that + +var teamDescription = Team of Robots +var teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here + +var teamName = process.env.GHES_TEAM_NAME +var teamAccess = 'pull' // pull,push,admin options here +var teamId = '' + +var orgRepos = [] + +// var creator = "" + +var foo = someFunction(); +var bar = a + 1; + +http.createServer(function (req, res) { + handler(req, res, function (err) { + console.log(err) + res.statusCode = 404 + res.end('no such location') + }) +}).listen(3000) + +handler.on('error', function (err) { + console.await.error('Error:', err.message) +}) + +handler.on('repository', function (event) { + if (event.payload.action === 'created') { + const repo = event.payload.repository.full_name + console.log(repo) + const org = event.payload.repository.owner.login + getTeamID(org) + setTimeout(checkTeamIDVariable, 1000) + } +}) + +handler.on('team', function (event) { +// TODO user events such as being removed from team or org + if (event.payload.action === 'deleted') { + // const name = event.payload.team.name + const org = event.payload.organization.login + getRepositories(org) + setTimeout(checkReposVariable, 5000) + } else if (event.payload.action === 'removed_from_repository') { + const org = event.payload.organization.login + getTeamID(org) + // const repo = event.payload.repository.full_name + setTimeout(checkTeamIDVariable, 1000) + } +}) + +function getTeamID (org) { + const https = require('https') + + const options = { + hostname: (process.env.GHE_HOST), + port: 443 + path: '/api/v3/orgs/' + org + '/teams', + method: 'GET', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json' + } + } + let body = [] + const req = https.request(options, (res) => { + res.on('data', (chunk) => { + body.push(chunk) + }).on('end', () => { + body = JSON.parse(Buffer.concat(body)) + body.forEach(item => { + if (item.name === teamName) { + teamId = item.id + } + }) + }) + }) + + req.on('error, (error) => { + console.error(error) + }) + + req.end() +} + +function checkTeamIDVariable (repo) { + if (typeof teamId != 'undefined') { + addTeamToRepo(repo, teamId) + } +} + +function checkReposVariable (org) { + if (typeof orgRepos !== 'undefined') { + // for(var repo of orgRepos) { + // addTeamToRepo(repo, teamId) + // } + reCreateTeam(org) + } +} + +function addTeamToRepo (repo, teamId) { + const https = require('https') + const data = JSON.stringify({ + permission: teamAccess + }) + + const options = { + hostname: (process.env.GHE_HOST), + port: 443, + path: '/api/v3/teams/' + teamId + '/repos/' + repo, + method: 'PUT', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json', + 'Content-Length': data.length + } + } + let body = [] + + const req = https.request(options, (res) => { + res.on('data', (chunk) => { + + body.push(chunk) + + }).on('end', () => { + + body = Buffer.concat(body).toString() + console.log(res.statusCode) + console.log('added team to ' + repo) + }) + }) + + req.on('error', (error) => { + console.error(error) + }) + + req.write(data) + req.end() +} + +function reCreateTeam (org) { + const https = require('https') + const data = JSON.stringify({ + name: teamName, + description: teamDescription, + privacy: teamPrivacy + maintainers: userArray, + repo_names: orgRepos + }) + + const options = { + hostname: (process.env.GHE_HOST), + port: 443 + path: '/api/v3/orgs/' + org + '/teams', + method: 'POST', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json', + 'Content-Length': data.length + } + } + // const body = [] + const req = https.request(options, (res) => { + if (res.statusCode !== 201) { + console.log('Status code: ' + res.statusCode) + console.log('Added ' + teamName + ' to ' + org + ' Failed') + res.on('data', function (chunk) { + console.log('BODY: ' + chunk) + }) + } else { + console.log('Added ' + teamName ' to ' + org) + } + }) + + req.on('error', (error) => { + console.error(error) + }) + + req.write(data) + req.end() +} + +function getRepositories (org) { + orgRepos = [] + + const https = require('https') + + const options = { + hostname: (process.env.GHE_HOST), + port: '443', + path: '/api/v3/orgs/' + org + "/repos", + method: 'GET', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json' + } + } + let body = [] + const req = https.request(options, (res) => { + res.on('data', (chunk) => { + body.push(chunk) + + }).on('end', () => { + body = JSON.parse(Buffer.concat(body)) + body.forEach(item => { + orgRepos.push(item.full_name) + + console.log(item.full_name) + }) + }) + }) + + req.on('error', (error) => { + console.error(error) + }) + req.end() +} diff --git a/.automation/test/sample_project_sarif/package-lock.json b/.automation/test/sample_project_sarif/package-lock.json new file mode 100644 index 00000000000..e1f3a29a229 --- /dev/null +++ b/.automation/test/sample_project_sarif/package-lock.json @@ -0,0 +1,138 @@ +{ + "name": "bad", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "tar": "^6.0.1" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.5.tgz", + "integrity": "sha512-+8NzxD82XQoNKNrl1d/FSi+X8wAEWR+sbYAfIvub4Nz0d22plFG72CEVVaufV8PNf4qSslFTD8VMOxNVhHCjTw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.0.1.tgz", + "integrity": "sha512-bKhKrrz2FJJj5s7wynxy/fyxpE0CmCjmOQ1KV4KkgXFWOgoIT/NbTMnB1n+LFNrNk0SSBVGGxcK5AGsyC+pW5Q==", + "dependencies": { + "chownr": "^1.1.3", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.0", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + }, + "dependencies": { + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "requires": { + "minipass": "^3.0.0" + } + }, + "minipass": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.5.tgz", + "integrity": "sha512-+8NzxD82XQoNKNrl1d/FSi+X8wAEWR+sbYAfIvub4Nz0d22plFG72CEVVaufV8PNf4qSslFTD8VMOxNVhHCjTw==", + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + }, + "tar": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.0.1.tgz", + "integrity": "sha512-bKhKrrz2FJJj5s7wynxy/fyxpE0CmCjmOQ1KV4KkgXFWOgoIT/NbTMnB1n+LFNrNk0SSBVGGxcK5AGsyC+pW5Q==", + "requires": { + "chownr": "^1.1.3", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.0", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } +} diff --git a/.automation/test/sample_project_sarif/package.json b/.automation/test/sample_project_sarif/package.json new file mode 100644 index 00000000000..d27eb0f729e --- /dev/null +++ b/.automation/test/sample_project_sarif/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "tar": "^6.0.1" + } +} diff --git a/.automation/test/sample_project_sarif/python_bad_1.py b/.automation/test/sample_project_sarif/python_bad_1.py new file mode 100644 index 00000000000..d2a9efdb2de --- /dev/null +++ b/.automation/test/sample_project_sarif/python_bad_1.py @@ -0,0 +1,4 @@ +try: + pass +except: + pass diff --git a/.automation/test/sample_project_sarif/terraform_checkov_bad_1.tf b/.automation/test/sample_project_sarif/terraform_checkov_bad_1.tf new file mode 100644 index 00000000000..c563d02bdae --- /dev/null +++ b/.automation/test/sample_project_sarif/terraform_checkov_bad_1.tf @@ -0,0 +1,3 @@ +resource "aws_secretsmanager_secret" "bad" { + name = "test" +} \ No newline at end of file diff --git a/.automation/test/sample_project_sarif/terraform_kics_bad_1.tf b/.automation/test/sample_project_sarif/terraform_kics_bad_1.tf new file mode 100644 index 00000000000..7f821843729 --- /dev/null +++ b/.automation/test/sample_project_sarif/terraform_kics_bad_1.tf @@ -0,0 +1,12 @@ +resource "aws_ami" "bad_example" { + name = "terraform-example" + virtualization_type = "hvm" + root_device_name = "/dev/xvda2" + + ebs_block_device { + device_name = "/dev/xvda2" + snapshot_id = "snap-xxxxxxxx" + volume_size = 8 + encrypted = false + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 3787ea96147..c2cb4cfad9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,20 +10,29 @@ Note: Can be used with `megalinter/megalinter@beta` in your GitHub Action mega-l -- Add [PMD](https://pmd.github.io/) to lint java files (disabled for now) -- Add [gitleaks](https://github.com/zricethezav/gitleaks) to lint git repository -- Add [goodcheck](https://github.com/sider/goodcheck) as regex-based linter -- Add [trivy](https://github.com/aquasecurity/trivy) security linter -- New flavor **Security** -- New descriptor **repository**: contains secretlint, git_diff, gitleaks and goodcheck (in next major versions, credentials and git descriptors will be deprecated, then removed) -- Manage offline run of `bash build.sh` for those who want to code in planes :) -- Automate update of CHANGELOG.md after release (pilot) -- New reporter **SARIF** -- SARIF management for: - - bandit - - checkov - - eslint -- Rename default report folder from `report` to `megalinter-reports` +- Core architecture + - New reporter **SARIF_REPORTER** that aggregates all SARIF output files into a single one + - Manage offline run of `bash build.sh` for those who want to code in planes :) + - Automate update of CHANGELOG.md after release (pilot) + - Rename default report folder from `report` to `megalinter-reports` + +- Linters: + - Add [PMD](https://pmd.github.io/) to lint java files (disabled for now) + - Add [gitleaks](https://github.com/zricethezav/gitleaks) to lint git repository + - Add [goodcheck](https://github.com/sider/goodcheck) as regex-based linter + - Add [trivy](https://github.com/aquasecurity/trivy) security linter + - SARIF management for: + - bandit + - checkov + - eslint + - gitleaks + +- Descriptors: + - New flavor **Security** + - New descriptor **repository**: contains secretlint, git_diff, gitleaks and goodcheck + - remove CREDENTIALS and GIT descriptors + + - Fix jscpd typo about `.venv` (#986) - markdownlint: rename default config file from .markdown-lint.json to .markdownlint.json diff --git a/megalinter/descriptors/repository.megalinter-descriptor.yml b/megalinter/descriptors/repository.megalinter-descriptor.yml index eb03fc60f80..761e8f2c3a4 100644 --- a/megalinter/descriptors/repository.megalinter-descriptor.yml +++ b/megalinter/descriptors/repository.megalinter-descriptor.yml @@ -57,6 +57,7 @@ linters: # GITLEAKS - linter_name: gitleaks + can_output_sarif: true descriptor_flavors: - all_flavors # Applicable to CI in any language project - ci_light @@ -69,6 +70,11 @@ linters: cli_lint_mode: project cli_lint_extra_args: - detect + cli_sarif_args: + - --report-format + - sarif + - --report-path + - "{{SARIF_OUTPUT_FILE}}" cli_lint_extra_args_after: - "--no-git" - "--verbose" diff --git a/megalinter/reporters/SarifReporter.py b/megalinter/reporters/SarifReporter.py index 63abca3c516..429c9248ee4 100644 --- a/megalinter/reporters/SarifReporter.py +++ b/megalinter/reporters/SarifReporter.py @@ -17,9 +17,7 @@ class SarifReporter(Reporter): def __init__(self, params=None): # Deactivate JSON output by default self.is_active = False - self.processing_order = ( - - 9999 # Run first - ) + self.processing_order = -9999 # Run first super().__init__(params) def manage_activation(self): @@ -30,15 +28,17 @@ def produce_report(self): sarif_obj = { "$schema": "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", "version": "2.1.0", - "properties": { - "comment": "Generated by MegaLinter" - }, - "runs": [] + "properties": {"comment": "Generated by MegaLinter"}, + "runs": [], } # Build unique SARIF file with all SARIF output files for linter in self.master.linters: - if linter.sarif_output_file is not None: - with open(linter.sarif_output_file, "r", encoding="utf-8") as linter_sarif_file: + if linter.sarif_output_file is not None and os.path.isfile( + linter.sarif_output_file + ): + with open( + linter.sarif_output_file, "r", encoding="utf-8" + ) as linter_sarif_file: linter_sarif_obj = json.load(linter_sarif_file) sarif_obj["runs"] += linter_sarif_obj["runs"] result_json = json.dumps(sarif_obj, sort_keys=True, indent=4) diff --git a/megalinter/tests/test_megalinter/helpers/utilstest.py b/megalinter/tests/test_megalinter/helpers/utilstest.py index acd4cfa7505..c56ac4d6e6b 100644 --- a/megalinter/tests/test_megalinter/helpers/utilstest.py +++ b/megalinter/tests/test_megalinter/helpers/utilstest.py @@ -14,6 +14,7 @@ from git import Repo from megalinter import Megalinter, config, utils +from megalinter.constants import DEFAULT_REPORT_FOLDER_NAME REPO_HOME = ( "/tmp/lint" @@ -253,7 +254,7 @@ def manage_copy_sources(workspace): def copy_logs_for_doc(text_report_file, test_folder, report_file_name): updated_sources_dir = ( f"{REPO_HOME}{os.path.sep}report{os.path.sep}updated_dev_sources{os.path.sep}" - f".automation{os.path.sep}test{os.path.sep}{test_folder}{os.path.sep}reports" + f".automation{os.path.sep}test{os.path.sep}{test_folder}{os.path.sep}{DEFAULT_REPORT_FOLDER_NAME}" ) target_file = f"{updated_sources_dir}{os.path.sep}{report_file_name}".replace( ".log", ".txt" @@ -399,13 +400,13 @@ def test_linter_report_tap(linter, test_self): f"expected-{linter.descriptor_id}.tap", ] + reports_with_extension for file_nm in list(dict.fromkeys(possible_reports)): - if os.path.isfile(f"{workspace}{os.path.sep}reports{os.path.sep}{file_nm}"): + if os.path.isfile(f"{workspace}{os.path.sep}{DEFAULT_REPORT_FOLDER_NAME}{os.path.sep}{file_nm}"): expected_file_name = ( - f"{workspace}{os.path.sep}reports{os.path.sep}{file_nm}" + f"{workspace}{os.path.sep}{DEFAULT_REPORT_FOLDER_NAME}{os.path.sep}{file_nm}" ) if expected_file_name == "": raise unittest.SkipTest( - f"Expected report not defined in {workspace}{os.path.sep}reports" + f"Expected report not defined in {workspace}{os.path.sep}{DEFAULT_REPORT_FOLDER_NAME}" ) # Call linter tmp_report_folder = tempfile.gettempdir() diff --git a/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py b/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py index 9e8871c1a05..ad51cb87626 100644 --- a/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py +++ b/megalinter/tests/test_megalinter/mega_linter_2_fixes_test.py @@ -10,7 +10,7 @@ from megalinter.tests.test_megalinter.helpers import utilstest -class mega_linter_2_fixes(unittest.TestCase): +class mega_linter_2_fixes_test(unittest.TestCase): def setUp(self): utilstest.linter_test_setup( { @@ -24,7 +24,8 @@ def test_1_apply_fixes_on_one_linter(self): "APPLY_FIXES": "JAVASCRIPT_STANDARD", "LOG_LEVEL": "DEBUG", "MULTI_STATUS": "false", - "DISABLE_LINTERS": "TERRAFORM_KICS", + "DISABLE_LINTERS": "TERRAFORM_KICS,REPOSITORY_GITLEAKS,REPOSITORY_TRIVY," + "JSON_V8R,YAML_V8R,MARKDOWN_MARKDOWN_LINK_CHECK,TERRAFORM_CHECKOV", } ) self.assertTrue( diff --git a/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py b/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py new file mode 100644 index 00000000000..38f049dc2a5 --- /dev/null +++ b/megalinter/tests/test_megalinter/mega_linter_3_sarif_test.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +""" +Unit tests for Megalinter class + +""" +import os +import unittest + +from megalinter.tests.test_megalinter.helpers import utilstest + + +class mega_linter_3_sarif_test(unittest.TestCase): + def setUp(self): + utilstest.linter_test_setup( + { + "sub_lint_root": f"{os.path.sep}.automation{os.path.sep}test{os.path.sep}sample_project_sarif" + } + ) + + def test_sarif_output(self): + mega_linter, output = utilstest.call_mega_linter( + { + "APPLY_FIXES": "false", + "LOG_LEVEL": "DEBUG", + "MULTI_STATUS": "false", + "ENABLE_LINTERS": "JAVASCRIPT_ES,REPOSITORY_TRIVY,REPOSITORY_GITLEAKS,PYTHON_BANDIT,TERRAFORM_KICS", + "SARIF_REPORTER": "true" + } + ) + self.assertTrue( + len(mega_linter.linters) > 0, "Linters have been created and run" + ) + expected_output_file = ( + mega_linter.report_folder + os.path.sep + "mega-linter-report.sarif" + ) + self.assertTrue( + os.path.isfile(expected_output_file), + "Output aggregated SARIF file " + expected_output_file + " should exist", + )